2024-11-16 09:57:53,136 main DEBUG Apache Log4j Core 2.17.2 initializing configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@47caedad 2024-11-16 09:57:53,148 main DEBUG Took 0.009796 seconds to load 1 plugins from package org.apache.hadoop.hbase.logging 2024-11-16 09:57:53,148 main DEBUG PluginManager 'Core' found 129 plugins 2024-11-16 09:57:53,148 main DEBUG PluginManager 'Level' found 0 plugins 2024-11-16 09:57:53,149 main DEBUG PluginManager 'Lookup' found 16 plugins 2024-11-16 09:57:53,151 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-16 09:57:53,158 main DEBUG PluginManager 'TypeConverter' found 26 plugins 2024-11-16 09:57:53,171 main DEBUG LoggerConfig$Builder(additivity="null", level="ERROR", levelAndRefs="null", name="org.apache.hadoop.metrics2.util.MBeans", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-16 09:57:53,172 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-16 09:57:53,173 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase.logging.TestJul2Slf4j", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-16 09:57:53,173 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-16 09:57:53,174 main DEBUG LoggerConfig$Builder(additivity="null", level="ERROR", levelAndRefs="null", name="org.apache.zookeeper", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-16 09:57:53,174 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-16 09:57:53,175 main DEBUG LoggerConfig$Builder(additivity="null", level="WARN", levelAndRefs="null", name="org.apache.hadoop.metrics2.impl.MetricsSinkAdapter", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-16 09:57:53,175 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-16 09:57:53,175 main DEBUG LoggerConfig$Builder(additivity="null", level="ERROR", levelAndRefs="null", name="org.apache.hadoop.metrics2.impl.MetricsSystemImpl", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-16 09:57:53,176 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-16 09:57:53,176 main DEBUG LoggerConfig$Builder(additivity="false", level="WARN", levelAndRefs="null", name="org.apache.directory", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-16 09:57:53,176 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-16 09:57:53,177 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase.ipc.FailedServers", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-16 09:57:53,177 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-16 09:57:53,178 main DEBUG LoggerConfig$Builder(additivity="null", level="WARN", levelAndRefs="null", name="org.apache.hadoop.metrics2.impl.MetricsConfig", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-16 09:57:53,178 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-16 09:57:53,178 main DEBUG LoggerConfig$Builder(additivity="null", level="INFO", levelAndRefs="null", name="org.apache.hadoop.hbase.ScheduledChore", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-16 09:57:53,178 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-16 09:57:53,179 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase.regionserver.RSRpcServices", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-16 09:57:53,179 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-16 09:57:53,180 main DEBUG LoggerConfig$Builder(additivity="null", level="WARN", levelAndRefs="null", name="org.apache.hadoop", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-16 09:57:53,180 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-16 09:57:53,180 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-16 09:57:53,181 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-16 09:57:53,181 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hbase.thirdparty.io.netty.channel", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-16 09:57:53,181 main DEBUG Building Plugin[name=root, class=org.apache.logging.log4j.core.config.LoggerConfig$RootLogger]. 2024-11-16 09:57:53,183 main DEBUG LoggerConfig$RootLogger$Builder(additivity="null", level="null", levelAndRefs="INFO,Console", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-16 09:57:53,184 main DEBUG Building Plugin[name=loggers, class=org.apache.logging.log4j.core.config.LoggersPlugin]. 2024-11-16 09:57:53,186 main DEBUG createLoggers(={org.apache.hadoop.metrics2.util.MBeans, org.apache.hadoop.hbase.logging.TestJul2Slf4j, org.apache.zookeeper, org.apache.hadoop.metrics2.impl.MetricsSinkAdapter, org.apache.hadoop.metrics2.impl.MetricsSystemImpl, org.apache.directory, org.apache.hadoop.hbase.ipc.FailedServers, org.apache.hadoop.metrics2.impl.MetricsConfig, org.apache.hadoop.hbase.ScheduledChore, org.apache.hadoop.hbase.regionserver.RSRpcServices, org.apache.hadoop, org.apache.hadoop.hbase, org.apache.hbase.thirdparty.io.netty.channel, root}) 2024-11-16 09:57:53,186 main DEBUG Building Plugin[name=layout, class=org.apache.logging.log4j.core.layout.PatternLayout]. 2024-11-16 09:57:53,188 main DEBUG PatternLayout$Builder(pattern="%d{ISO8601} %-5p [%t%notEmpty{ %X}] %C{2}(%L): %m%n", PatternSelector=null, Configuration(PropertiesConfig), Replace=null, charset="null", alwaysWriteExceptions="null", disableAnsi="null", noConsoleNoAnsi="null", header="null", footer="null") 2024-11-16 09:57:53,188 main DEBUG PluginManager 'Converter' found 47 plugins 2024-11-16 09:57:53,196 main DEBUG Building Plugin[name=appender, class=org.apache.hadoop.hbase.logging.HBaseTestAppender]. 2024-11-16 09:57:53,199 main DEBUG HBaseTestAppender$Builder(target="SYSTEM_ERR", maxSize="1G", bufferedIo="null", bufferSize="null", immediateFlush="null", ignoreExceptions="null", PatternLayout(%d{ISO8601} %-5p [%t%notEmpty{ %X}] %C{2}(%L): %m%n), name="Console", Configuration(PropertiesConfig), Filter=null, ={}) 2024-11-16 09:57:53,201 main DEBUG Starting HBaseTestOutputStreamManager SYSTEM_ERR 2024-11-16 09:57:53,201 main DEBUG Building Plugin[name=appenders, class=org.apache.logging.log4j.core.config.AppendersPlugin]. 2024-11-16 09:57:53,201 main DEBUG createAppenders(={Console}) 2024-11-16 09:57:53,202 main DEBUG Configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@47caedad initialized 2024-11-16 09:57:53,202 main DEBUG Starting configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@47caedad 2024-11-16 09:57:53,203 main DEBUG Started configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@47caedad OK. 2024-11-16 09:57:53,203 main DEBUG Shutting down OutputStreamManager SYSTEM_OUT.false.false-1 2024-11-16 09:57:53,203 main DEBUG OutputStream closed 2024-11-16 09:57:53,204 main DEBUG Shut down OutputStreamManager SYSTEM_OUT.false.false-1, all resources released: true 2024-11-16 09:57:53,204 main DEBUG Appender DefaultConsole-1 stopped with status true 2024-11-16 09:57:53,204 main DEBUG Stopped org.apache.logging.log4j.core.config.DefaultConfiguration@55634720 OK 2024-11-16 09:57:53,273 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6 2024-11-16 09:57:53,275 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=StatusLogger 2024-11-16 09:57:53,276 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=ContextSelector 2024-11-16 09:57:53,277 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name= 2024-11-16 09:57:53,278 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.directory 2024-11-16 09:57:53,278 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.impl.MetricsSinkAdapter 2024-11-16 09:57:53,278 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.zookeeper 2024-11-16 09:57:53,279 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.logging.TestJul2Slf4j 2024-11-16 09:57:53,279 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.impl.MetricsSystemImpl 2024-11-16 09:57:53,279 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.util.MBeans 2024-11-16 09:57:53,279 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase 2024-11-16 09:57:53,280 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop 2024-11-16 09:57:53,280 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.ipc.FailedServers 2024-11-16 09:57:53,280 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.regionserver.RSRpcServices 2024-11-16 09:57:53,281 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.impl.MetricsConfig 2024-11-16 09:57:53,281 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hbase.thirdparty.io.netty.channel 2024-11-16 09:57:53,281 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.ScheduledChore 2024-11-16 09:57:53,282 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Appenders,name=Console 2024-11-16 09:57:53,284 main DEBUG org.apache.logging.log4j.core.util.SystemClock supports precise timestamps. 2024-11-16 09:57:53,284 main DEBUG Reconfiguration complete for context[name=1dbd16a6] at URI jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-logging/target/hbase-logging-4.0.0-alpha-1-SNAPSHOT-tests.jar!/log4j2.properties (org.apache.logging.log4j.core.LoggerContext@19835e64) with optional ClassLoader: null 2024-11-16 09:57:53,285 main DEBUG Shutdown hook enabled. Registering a new one. 2024-11-16 09:57:53,285 main DEBUG LoggerContext[name=1dbd16a6, org.apache.logging.log4j.core.LoggerContext@19835e64] started OK. 2024-11-16T09:57:53,529 DEBUG [main {}] hbase.HBaseTestingUtil(323): Setting hbase.rootdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/62476c2b-1e2d-7029-5a2f-94eb6a0fedba 2024-11-16 09:57:53,532 main DEBUG AsyncLogger.ThreadNameStrategy=UNCACHED (user specified null, default is UNCACHED) 2024-11-16 09:57:53,532 main DEBUG org.apache.logging.log4j.core.util.SystemClock supports precise timestamps. 2024-11-16T09:57:53,542 INFO [main {}] hbase.HBaseClassTestRule(94): Test class org.apache.hadoop.hbase.TestAcidGuaranteesWithNoInMemCompaction timeout: 13 mins 2024-11-16T09:57:53,568 INFO [Time-limited test {}] hbase.HBaseTestingUtil(805): Starting up minicluster with option: StartMiniClusterOption{numMasters=1, masterClass=null, numRegionServers=1, rsPorts=, rsClass=null, numDataNodes=1, dataNodeHosts=null, numZkServers=1, createRootDir=false, createWALDir=false} 2024-11-16T09:57:53,584 INFO [Time-limited test {}] hbase.HBaseZKTestingUtil(84): Created new mini-cluster data directory: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/62476c2b-1e2d-7029-5a2f-94eb6a0fedba/cluster_7e615273-6ed6-1343-5769-ac1955575909, deleteOnExit=true 2024-11-16T09:57:53,584 INFO [Time-limited test {}] hbase.HBaseTestingUtil(818): STARTING DFS 2024-11-16T09:57:53,585 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting test.cache.data to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/62476c2b-1e2d-7029-5a2f-94eb6a0fedba/test.cache.data in system properties and HBase conf 2024-11-16T09:57:53,586 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting hadoop.tmp.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/62476c2b-1e2d-7029-5a2f-94eb6a0fedba/hadoop.tmp.dir in system properties and HBase conf 2024-11-16T09:57:53,586 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting hadoop.log.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/62476c2b-1e2d-7029-5a2f-94eb6a0fedba/hadoop.log.dir in system properties and HBase conf 2024-11-16T09:57:53,587 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting mapreduce.cluster.local.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/62476c2b-1e2d-7029-5a2f-94eb6a0fedba/mapreduce.cluster.local.dir in system properties and HBase conf 2024-11-16T09:57:53,588 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting mapreduce.cluster.temp.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/62476c2b-1e2d-7029-5a2f-94eb6a0fedba/mapreduce.cluster.temp.dir in system properties and HBase conf 2024-11-16T09:57:53,588 INFO [Time-limited test {}] hbase.HBaseTestingUtil(738): read short circuit is OFF 2024-11-16T09:57:53,689 WARN [Time-limited test {}] util.NativeCodeLoader(60): Unable to load native-hadoop library for your platform... using builtin-java classes where applicable 2024-11-16T09:57:53,787 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-11-16T09:57:53,790 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.node-labels.fs-store.root-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/62476c2b-1e2d-7029-5a2f-94eb6a0fedba/yarn.node-labels.fs-store.root-dir in system properties and HBase conf 2024-11-16T09:57:53,791 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.node-attribute.fs-store.root-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/62476c2b-1e2d-7029-5a2f-94eb6a0fedba/yarn.node-attribute.fs-store.root-dir in system properties and HBase conf 2024-11-16T09:57:53,791 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.nodemanager.log-dirs to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/62476c2b-1e2d-7029-5a2f-94eb6a0fedba/yarn.nodemanager.log-dirs in system properties and HBase conf 2024-11-16T09:57:53,792 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.nodemanager.remote-app-log-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/62476c2b-1e2d-7029-5a2f-94eb6a0fedba/yarn.nodemanager.remote-app-log-dir in system properties and HBase conf 2024-11-16T09:57:53,792 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.timeline-service.entity-group-fs-store.active-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/62476c2b-1e2d-7029-5a2f-94eb6a0fedba/yarn.timeline-service.entity-group-fs-store.active-dir in system properties and HBase conf 2024-11-16T09:57:53,793 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.timeline-service.entity-group-fs-store.done-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/62476c2b-1e2d-7029-5a2f-94eb6a0fedba/yarn.timeline-service.entity-group-fs-store.done-dir in system properties and HBase conf 2024-11-16T09:57:53,793 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.nodemanager.remote-app-log-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/62476c2b-1e2d-7029-5a2f-94eb6a0fedba/yarn.nodemanager.remote-app-log-dir in system properties and HBase conf 2024-11-16T09:57:53,794 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.journalnode.edits.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/62476c2b-1e2d-7029-5a2f-94eb6a0fedba/dfs.journalnode.edits.dir in system properties and HBase conf 2024-11-16T09:57:53,794 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.datanode.shared.file.descriptor.paths to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/62476c2b-1e2d-7029-5a2f-94eb6a0fedba/dfs.datanode.shared.file.descriptor.paths in system properties and HBase conf 2024-11-16T09:57:53,794 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting nfs.dump.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/62476c2b-1e2d-7029-5a2f-94eb6a0fedba/nfs.dump.dir in system properties and HBase conf 2024-11-16T09:57:53,795 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting java.io.tmpdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/62476c2b-1e2d-7029-5a2f-94eb6a0fedba/java.io.tmpdir in system properties and HBase conf 2024-11-16T09:57:53,795 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.journalnode.edits.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/62476c2b-1e2d-7029-5a2f-94eb6a0fedba/dfs.journalnode.edits.dir in system properties and HBase conf 2024-11-16T09:57:53,795 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.provided.aliasmap.inmemory.leveldb.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/62476c2b-1e2d-7029-5a2f-94eb6a0fedba/dfs.provided.aliasmap.inmemory.leveldb.dir in system properties and HBase conf 2024-11-16T09:57:53,796 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting fs.s3a.committer.staging.tmp.path to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/62476c2b-1e2d-7029-5a2f-94eb6a0fedba/fs.s3a.committer.staging.tmp.path in system properties and HBase conf 2024-11-16T09:57:54,607 WARN [Time-limited test {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-namenode.properties,hadoop-metrics2.properties 2024-11-16T09:57:54,686 INFO [Time-limited test {}] log.Log(170): Logging initialized @2323ms to org.eclipse.jetty.util.log.Slf4jLog 2024-11-16T09:57:54,768 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-11-16T09:57:54,833 INFO [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9 2024-11-16T09:57:54,856 INFO [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0 2024-11-16T09:57:54,857 INFO [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults 2024-11-16T09:57:54,858 INFO [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 660000ms 2024-11-16T09:57:54,872 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-11-16T09:57:54,875 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@6b23711f{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/62476c2b-1e2d-7029-5a2f-94eb6a0fedba/hadoop.log.dir/,AVAILABLE} 2024-11-16T09:57:54,876 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@93c62df{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE} 2024-11-16T09:57:55,084 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@4ded99d3{hdfs,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/62476c2b-1e2d-7029-5a2f-94eb6a0fedba/java.io.tmpdir/jetty-localhost-46147-hadoop-hdfs-3_4_1-tests_jar-_-any-6688314587811919168/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/hdfs} 2024-11-16T09:57:55,093 INFO [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@557285fa{HTTP/1.1, (http/1.1)}{localhost:46147} 2024-11-16T09:57:55,093 INFO [Time-limited test {}] server.Server(415): Started @2731ms 2024-11-16T09:57:55,483 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-11-16T09:57:55,490 INFO [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9 2024-11-16T09:57:55,491 INFO [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0 2024-11-16T09:57:55,491 INFO [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults 2024-11-16T09:57:55,492 INFO [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 600000ms 2024-11-16T09:57:55,493 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@4fd7fa7c{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/62476c2b-1e2d-7029-5a2f-94eb6a0fedba/hadoop.log.dir/,AVAILABLE} 2024-11-16T09:57:55,493 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@435f9f96{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE} 2024-11-16T09:57:55,617 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@4269bd31{datanode,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/62476c2b-1e2d-7029-5a2f-94eb6a0fedba/java.io.tmpdir/jetty-localhost-35533-hadoop-hdfs-3_4_1-tests_jar-_-any-1284940602522451282/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode} 2024-11-16T09:57:55,619 INFO [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@353d64f2{HTTP/1.1, (http/1.1)}{localhost:35533} 2024-11-16T09:57:55,619 INFO [Time-limited test {}] server.Server(415): Started @3256ms 2024-11-16T09:57:55,679 WARN [Time-limited test {}] web.RestCsrfPreventionFilterHandler(75): Got null for restCsrfPreventionFilter - will not do any filtering. 2024-11-16T09:57:56,127 WARN [Thread-72 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/62476c2b-1e2d-7029-5a2f-94eb6a0fedba/cluster_7e615273-6ed6-1343-5769-ac1955575909/data/data1/current/BP-1247346184-172.17.0.2-1731751074355/current, will proceed with Du for space computation calculation, 2024-11-16T09:57:56,127 WARN [Thread-73 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/62476c2b-1e2d-7029-5a2f-94eb6a0fedba/cluster_7e615273-6ed6-1343-5769-ac1955575909/data/data2/current/BP-1247346184-172.17.0.2-1731751074355/current, will proceed with Du for space computation calculation, 2024-11-16T09:57:56,169 WARN [Thread-58 {}] datanode.DirectoryScanner(302): dfs.datanode.directoryscan.throttle.limit.ms.per.sec set to value above 1000 ms/sec. Assuming default value of -1 2024-11-16T09:57:56,221 INFO [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0x152a778f0261c727 with lease ID 0xae50fdb80cc1d714: Processing first storage report for DS-93631843-23e1-4c59-bec2-65fdae3c8e7b from datanode DatanodeRegistration(127.0.0.1:37591, datanodeUuid=347a0a29-1ecd-4397-8217-aa195b8a4ba6, infoPort=36709, infoSecurePort=0, ipcPort=38337, storageInfo=lv=-57;cid=testClusterID;nsid=1258990090;c=1731751074355) 2024-11-16T09:57:56,223 INFO [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0x152a778f0261c727 with lease ID 0xae50fdb80cc1d714: from storage DS-93631843-23e1-4c59-bec2-65fdae3c8e7b node DatanodeRegistration(127.0.0.1:37591, datanodeUuid=347a0a29-1ecd-4397-8217-aa195b8a4ba6, infoPort=36709, infoSecurePort=0, ipcPort=38337, storageInfo=lv=-57;cid=testClusterID;nsid=1258990090;c=1731751074355), blocks: 0, hasStaleStorage: true, processing time: 1 msecs, invalidatedBlocks: 0 2024-11-16T09:57:56,223 INFO [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0x152a778f0261c727 with lease ID 0xae50fdb80cc1d714: Processing first storage report for DS-f9c203c2-a992-40e5-88d9-ed17b01ab34a from datanode DatanodeRegistration(127.0.0.1:37591, datanodeUuid=347a0a29-1ecd-4397-8217-aa195b8a4ba6, infoPort=36709, infoSecurePort=0, ipcPort=38337, storageInfo=lv=-57;cid=testClusterID;nsid=1258990090;c=1731751074355) 2024-11-16T09:57:56,224 INFO [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0x152a778f0261c727 with lease ID 0xae50fdb80cc1d714: from storage DS-f9c203c2-a992-40e5-88d9-ed17b01ab34a node DatanodeRegistration(127.0.0.1:37591, datanodeUuid=347a0a29-1ecd-4397-8217-aa195b8a4ba6, infoPort=36709, infoSecurePort=0, ipcPort=38337, storageInfo=lv=-57;cid=testClusterID;nsid=1258990090;c=1731751074355), blocks: 0, hasStaleStorage: false, processing time: 0 msecs, invalidatedBlocks: 0 2024-11-16T09:57:56,238 DEBUG [Time-limited test {}] hbase.HBaseTestingUtil(631): Setting hbase.rootdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/62476c2b-1e2d-7029-5a2f-94eb6a0fedba 2024-11-16T09:57:56,319 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(261): Started connectionTimeout=30000, dir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/62476c2b-1e2d-7029-5a2f-94eb6a0fedba/cluster_7e615273-6ed6-1343-5769-ac1955575909/zookeeper_0, clientPort=49647, secureClientPort=-1, dataDir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/62476c2b-1e2d-7029-5a2f-94eb6a0fedba/cluster_7e615273-6ed6-1343-5769-ac1955575909/zookeeper_0/version-2, dataDirSize=457 dataLogDir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/62476c2b-1e2d-7029-5a2f-94eb6a0fedba/cluster_7e615273-6ed6-1343-5769-ac1955575909/zookeeper_0/version-2, dataLogSize=457 tickTime=2000, maxClientCnxns=300, minSessionTimeout=4000, maxSessionTimeout=40000, clientPortListenBacklog=-1, serverId=0 2024-11-16T09:57:56,331 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(286): Started MiniZooKeeperCluster and ran 'stat' on client port=49647 2024-11-16T09:57:56,342 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-16T09:57:56,345 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-16T09:57:56,574 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37591 is added to blk_1073741825_1001 (size=7) 2024-11-16T09:57:56,982 INFO [Time-limited test {}] util.FSUtils(489): Created version file at hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592 with version=8 2024-11-16T09:57:56,982 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1139): Setting hbase.fs.tmp.dir to hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/hbase-staging 2024-11-16T09:57:57,077 DEBUG [Time-limited test {}] channel.MultithreadEventLoopGroup(44): -Dio.netty.eventLoopThreads: 16 2024-11-16T09:57:57,328 INFO [Time-limited test {}] client.ConnectionUtils(128): master/037c58a19ddb:0 server-side Connection retries=45 2024-11-16T09:57:57,339 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-16T09:57:57,339 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-11-16T09:57:57,344 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-11-16T09:57:57,344 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-16T09:57:57,344 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-11-16T09:57:57,481 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.MasterService, hbase.pb.RegionServerStatusService, hbase.pb.LockService, hbase.pb.HbckService, hbase.pb.ClientMetaService, hbase.pb.AdminService 2024-11-16T09:57:57,542 INFO [Time-limited test {}] metrics.MetricRegistriesLoader(60): Loaded MetricRegistries class org.apache.hadoop.hbase.metrics.impl.MetricRegistriesImpl 2024-11-16T09:57:57,551 DEBUG [Time-limited test {}] util.ClassSize(228): Using Unsafe to estimate memory layout 2024-11-16T09:57:57,555 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-11-16T09:57:57,583 DEBUG [Time-limited test {}] channel.DefaultChannelId(84): -Dio.netty.processId: 6459 (auto-detected) 2024-11-16T09:57:57,584 DEBUG [Time-limited test {}] channel.DefaultChannelId(106): -Dio.netty.machineId: 02:42:ac:ff:fe:11:00:02 (auto-detected) 2024-11-16T09:57:57,606 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.2:37877 2024-11-16T09:57:57,628 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=master:37877 connecting to ZooKeeper ensemble=127.0.0.1:49647 2024-11-16T09:57:57,664 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:378770x0, quorum=127.0.0.1:49647, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-11-16T09:57:57,667 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): master:37877-0x101b083d7320000 connected 2024-11-16T09:57:57,703 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-16T09:57:57,707 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-16T09:57:57,720 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): master:37877-0x101b083d7320000, quorum=127.0.0.1:49647, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-16T09:57:57,724 INFO [Time-limited test {}] master.HMaster(525): hbase.rootdir=hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592, hbase.cluster.distributed=false 2024-11-16T09:57:57,751 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): master:37877-0x101b083d7320000, quorum=127.0.0.1:49647, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl 2024-11-16T09:57:57,757 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=37877 2024-11-16T09:57:57,757 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=37877 2024-11-16T09:57:57,758 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=37877 2024-11-16T09:57:57,759 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=37877 2024-11-16T09:57:57,759 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=37877 2024-11-16T09:57:57,883 INFO [Time-limited test {}] client.ConnectionUtils(128): regionserver/037c58a19ddb:0 server-side Connection retries=45 2024-11-16T09:57:57,885 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-16T09:57:57,885 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-11-16T09:57:57,885 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-11-16T09:57:57,886 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-16T09:57:57,886 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-11-16T09:57:57,889 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.ClientService, hbase.pb.AdminService, hbase.pb.ClientMetaService, hbase.pb.BootstrapNodeService 2024-11-16T09:57:57,891 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-11-16T09:57:57,893 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.2:37267 2024-11-16T09:57:57,895 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=regionserver:37267 connecting to ZooKeeper ensemble=127.0.0.1:49647 2024-11-16T09:57:57,897 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-16T09:57:57,901 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-16T09:57:57,909 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:372670x0, quorum=127.0.0.1:49647, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-11-16T09:57:57,910 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:372670x0, quorum=127.0.0.1:49647, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-16T09:57:57,910 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): regionserver:37267-0x101b083d7320001 connected 2024-11-16T09:57:57,915 INFO [Time-limited test {}] hfile.BlockCacheFactory(123): Allocating BlockCache size=880 MB, blockSize=64 KB 2024-11-16T09:57:57,926 DEBUG [Time-limited test {}] mob.MobFileCache(124): MobFileCache enabled with cacheSize=1000, evictPeriods=3600sec, evictRemainRatio=0.5 2024-11-16T09:57:57,929 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:37267-0x101b083d7320001, quorum=127.0.0.1:49647, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/master 2024-11-16T09:57:57,935 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:37267-0x101b083d7320001, quorum=127.0.0.1:49647, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl 2024-11-16T09:57:57,935 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=37267 2024-11-16T09:57:57,936 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=37267 2024-11-16T09:57:57,939 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=37267 2024-11-16T09:57:57,940 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=37267 2024-11-16T09:57:57,941 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=37267 2024-11-16T09:57:57,957 DEBUG [M:0;037c58a19ddb:37877 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:M:0;037c58a19ddb:37877 2024-11-16T09:57:57,958 INFO [master/037c58a19ddb:0:becomeActiveMaster {}] master.HMaster(2510): Adding backup master ZNode /hbase/backup-masters/037c58a19ddb,37877,1731751077131 2024-11-16T09:57:57,965 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:37267-0x101b083d7320001, quorum=127.0.0.1:49647, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-16T09:57:57,965 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:37877-0x101b083d7320000, quorum=127.0.0.1:49647, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-16T09:57:57,967 DEBUG [master/037c58a19ddb:0:becomeActiveMaster {}] zookeeper.ZKUtil(111): master:37877-0x101b083d7320000, quorum=127.0.0.1:49647, baseZNode=/hbase Set watcher on existing znode=/hbase/backup-masters/037c58a19ddb,37877,1731751077131 2024-11-16T09:57:57,987 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:37267-0x101b083d7320001, quorum=127.0.0.1:49647, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/master 2024-11-16T09:57:57,987 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:37877-0x101b083d7320000, quorum=127.0.0.1:49647, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-16T09:57:57,987 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:37267-0x101b083d7320001, quorum=127.0.0.1:49647, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-16T09:57:57,989 DEBUG [master/037c58a19ddb:0:becomeActiveMaster {}] zookeeper.ZKUtil(111): master:37877-0x101b083d7320000, quorum=127.0.0.1:49647, baseZNode=/hbase Set watcher on existing znode=/hbase/master 2024-11-16T09:57:57,990 INFO [master/037c58a19ddb:0:becomeActiveMaster {}] master.ActiveMasterManager(245): Deleting ZNode for /hbase/backup-masters/037c58a19ddb,37877,1731751077131 from backup master directory 2024-11-16T09:57:57,993 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:37877-0x101b083d7320000, quorum=127.0.0.1:49647, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/backup-masters/037c58a19ddb,37877,1731751077131 2024-11-16T09:57:57,993 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:37267-0x101b083d7320001, quorum=127.0.0.1:49647, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-16T09:57:57,993 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:37877-0x101b083d7320000, quorum=127.0.0.1:49647, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-16T09:57:57,993 WARN [master/037c58a19ddb:0:becomeActiveMaster {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-11-16T09:57:57,994 INFO [master/037c58a19ddb:0:becomeActiveMaster {}] master.ActiveMasterManager(255): Registered as active master=037c58a19ddb,37877,1731751077131 2024-11-16T09:57:57,996 INFO [master/037c58a19ddb:0:becomeActiveMaster {}] regionserver.ChunkCreator(488): Allocating data MemStoreChunkPool with chunk size 2 MB, max count 396, initial count 0 2024-11-16T09:57:57,998 INFO [master/037c58a19ddb:0:becomeActiveMaster {}] regionserver.ChunkCreator(488): Allocating index MemStoreChunkPool with chunk size 204.80 KB, max count 440, initial count 0 2024-11-16T09:57:58,054 DEBUG [master/037c58a19ddb:0:becomeActiveMaster {}] util.FSUtils(620): Create cluster ID file [hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/hbase.id] with ID: 1fe46ee8-cb44-44b1-8251-37103629f786 2024-11-16T09:57:58,054 DEBUG [master/037c58a19ddb:0:becomeActiveMaster {}] util.FSUtils(625): Write the cluster ID file to a temporary location: hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/.tmp/hbase.id 2024-11-16T09:57:58,064 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37591 is added to blk_1073741826_1002 (size=42) 2024-11-16T09:57:58,466 DEBUG [master/037c58a19ddb:0:becomeActiveMaster {}] util.FSUtils(634): Move the temporary cluster ID file to its target location [hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/.tmp/hbase.id]:[hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/hbase.id] 2024-11-16T09:57:58,516 INFO [master/037c58a19ddb:0:becomeActiveMaster {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-16T09:57:58,522 INFO [master/037c58a19ddb:0:becomeActiveMaster {}] util.FSTableDescriptors(270): Fetching table descriptors from the filesystem. 2024-11-16T09:57:58,547 INFO [master/037c58a19ddb:0:becomeActiveMaster {}] util.FSTableDescriptors(299): Fetched table descriptors(size=0) cost 22ms. 2024-11-16T09:57:58,551 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:37877-0x101b083d7320000, quorum=127.0.0.1:49647, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-16T09:57:58,551 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:37267-0x101b083d7320001, quorum=127.0.0.1:49647, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-16T09:57:58,567 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37591 is added to blk_1073741827_1003 (size=196) 2024-11-16T09:57:58,985 INFO [master/037c58a19ddb:0:becomeActiveMaster {}] region.MasterRegion(370): Create or load local region for table 'master:store', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-16T09:57:58,987 INFO [master/037c58a19ddb:0:becomeActiveMaster {}] region.MasterRegionFlusherAndCompactor(132): Injected flushSize=134217728, flushPerChanges=1000000, flushIntervalMs=900000 2024-11-16T09:57:59,004 DEBUG [master/037c58a19ddb:0:becomeActiveMaster {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(244): No decryptEncryptedDataEncryptionKey method in DFSClient, should be hadoop version with HDFS-12396 java.lang.NoSuchMethodException: org.apache.hadoop.hdfs.DFSClient.decryptEncryptedDataEncryptionKey(org.apache.hadoop.fs.FileEncryptionInfo) at java.lang.Class.getDeclaredMethod(Class.java:2675) ~[?:?] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper.createTransparentCryptoHelperWithoutHDFS12396(FanOutOneBlockAsyncDFSOutputSaslHelper.java:183) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper.createTransparentCryptoHelper(FanOutOneBlockAsyncDFSOutputSaslHelper.java:242) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper.(FanOutOneBlockAsyncDFSOutputSaslHelper.java:253) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.lang.Class.forName0(Native Method) ~[?:?] at java.lang.Class.forName(Class.java:375) ~[?:?] at org.apache.hadoop.hbase.wal.AsyncFSWALProvider.load(AsyncFSWALProvider.java:150) ~[classes/:?] at org.apache.hadoop.hbase.wal.WALFactory.getProviderClass(WALFactory.java:174) ~[classes/:?] at org.apache.hadoop.hbase.wal.WALFactory.(WALFactory.java:262) ~[classes/:?] at org.apache.hadoop.hbase.wal.WALFactory.(WALFactory.java:231) ~[classes/:?] at org.apache.hadoop.hbase.master.region.MasterRegion.create(MasterRegion.java:400) ~[classes/:?] at org.apache.hadoop.hbase.master.region.MasterRegionFactory.create(MasterRegionFactory.java:135) ~[classes/:?] at org.apache.hadoop.hbase.master.HMaster.finishActiveMasterInitialization(HMaster.java:1003) ~[classes/:?] at org.apache.hadoop.hbase.master.HMaster.startActiveMasterManager(HMaster.java:2535) ~[classes/:?] at org.apache.hadoop.hbase.master.HMaster.lambda$run$0(HMaster.java:613) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.lambda$tracedRunnable$2(TraceUtil.java:155) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T09:57:59,009 INFO [master/037c58a19ddb:0:becomeActiveMaster {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-16T09:57:59,037 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37591 is added to blk_1073741828_1004 (size=1189) 2024-11-16T09:57:59,458 INFO [master/037c58a19ddb:0:becomeActiveMaster {}] regionserver.HRegion(7590): Creating {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='master:store', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, under table dir hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/MasterData/data/master/store 2024-11-16T09:57:59,476 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37591 is added to blk_1073741829_1005 (size=34) 2024-11-16T09:57:59,882 INFO [master/037c58a19ddb:0:becomeActiveMaster {}] throttle.StoreHotnessProtector(112): StoreHotnessProtector is disabled. Set hbase.region.store.parallel.put.limit > 0 to enable, which may help mitigate load under heavy write pressure. 2024-11-16T09:57:59,886 DEBUG [master/037c58a19ddb:0:becomeActiveMaster {}] regionserver.HRegion(898): Instantiated master:store,,1.1595e783b53d99cd5eef43b6debb2682.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-16T09:57:59,887 DEBUG [master/037c58a19ddb:0:becomeActiveMaster {}] regionserver.HRegion(1722): Closing 1595e783b53d99cd5eef43b6debb2682, disabling compactions & flushes 2024-11-16T09:57:59,887 INFO [master/037c58a19ddb:0:becomeActiveMaster {}] regionserver.HRegion(1755): Closing region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-16T09:57:59,887 DEBUG [master/037c58a19ddb:0:becomeActiveMaster {}] regionserver.HRegion(1776): Time limited wait for close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-16T09:57:59,889 DEBUG [master/037c58a19ddb:0:becomeActiveMaster {}] regionserver.HRegion(1843): Acquired close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. after waiting 0 ms 2024-11-16T09:57:59,889 DEBUG [master/037c58a19ddb:0:becomeActiveMaster {}] regionserver.HRegion(1853): Updates disabled for region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-16T09:57:59,889 INFO [master/037c58a19ddb:0:becomeActiveMaster {}] regionserver.HRegion(1973): Closed master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-16T09:57:59,890 DEBUG [master/037c58a19ddb:0:becomeActiveMaster {}] regionserver.HRegion(1676): Region close journal for 1595e783b53d99cd5eef43b6debb2682: Waiting for close lock at 1731751079887Disabling compacts and flushes for region at 1731751079887Disabling writes for close at 1731751079889 (+2 ms)Writing region close event to WAL at 1731751079889Closed at 1731751079889 2024-11-16T09:57:59,893 WARN [master/037c58a19ddb:0:becomeActiveMaster {}] region.MasterRegion(249): failed to clean up initializing flag: hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/MasterData/data/master/store/.initializing 2024-11-16T09:57:59,893 DEBUG [master/037c58a19ddb:0:becomeActiveMaster {}] region.MasterRegion(219): WALDir=hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/MasterData/WALs/037c58a19ddb,37877,1731751077131 2024-11-16T09:57:59,902 INFO [master/037c58a19ddb:0:becomeActiveMaster {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-11-16T09:57:59,918 INFO [master/037c58a19ddb:0:becomeActiveMaster {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=037c58a19ddb%2C37877%2C1731751077131, suffix=, logDir=hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/MasterData/WALs/037c58a19ddb,37877,1731751077131, archiveDir=hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/MasterData/oldWALs, maxLogs=10 2024-11-16T09:57:59,941 DEBUG [master/037c58a19ddb:0:becomeActiveMaster {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/MasterData/WALs/037c58a19ddb,37877,1731751077131/037c58a19ddb%2C37877%2C1731751077131.1731751079922, exclude list is [], retry=0 2024-11-16T09:57:59,958 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:37591,DS-93631843-23e1-4c59-bec2-65fdae3c8e7b,DISK] 2024-11-16T09:57:59,961 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] asyncfs.ProtobufDecoder(117): Hadoop 3.3 and above shades protobuf. 2024-11-16T09:57:59,997 INFO [master/037c58a19ddb:0:becomeActiveMaster {}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/MasterData/WALs/037c58a19ddb,37877,1731751077131/037c58a19ddb%2C37877%2C1731751077131.1731751079922 2024-11-16T09:57:59,998 DEBUG [master/037c58a19ddb:0:becomeActiveMaster {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:36709:36709)] 2024-11-16T09:57:59,998 DEBUG [master/037c58a19ddb:0:becomeActiveMaster {}] regionserver.HRegion(7752): Opening region: {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''} 2024-11-16T09:57:59,999 DEBUG [master/037c58a19ddb:0:becomeActiveMaster {}] regionserver.HRegion(898): Instantiated master:store,,1.1595e783b53d99cd5eef43b6debb2682.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-16T09:58:00,002 DEBUG [master/037c58a19ddb:0:becomeActiveMaster {}] regionserver.HRegion(7794): checking encryption for 1595e783b53d99cd5eef43b6debb2682 2024-11-16T09:58:00,003 DEBUG [master/037c58a19ddb:0:becomeActiveMaster {}] regionserver.HRegion(7797): checking classloading for 1595e783b53d99cd5eef43b6debb2682 2024-11-16T09:58:00,042 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1595e783b53d99cd5eef43b6debb2682 2024-11-16T09:58:00,071 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName info 2024-11-16T09:58:00,076 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T09:58:00,079 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-16T09:58:00,080 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family proc of region 1595e783b53d99cd5eef43b6debb2682 2024-11-16T09:58:00,084 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName proc 2024-11-16T09:58:00,084 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T09:58:00,085 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/proc, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-16T09:58:00,086 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rs of region 1595e783b53d99cd5eef43b6debb2682 2024-11-16T09:58:00,088 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName rs 2024-11-16T09:58:00,088 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T09:58:00,089 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/rs, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-16T09:58:00,090 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family state of region 1595e783b53d99cd5eef43b6debb2682 2024-11-16T09:58:00,092 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName state 2024-11-16T09:58:00,092 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T09:58:00,093 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/state, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-16T09:58:00,094 DEBUG [master/037c58a19ddb:0:becomeActiveMaster {}] regionserver.HRegion(1038): replaying wal for 1595e783b53d99cd5eef43b6debb2682 2024-11-16T09:58:00,097 DEBUG [master/037c58a19ddb:0:becomeActiveMaster {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682 2024-11-16T09:58:00,098 DEBUG [master/037c58a19ddb:0:becomeActiveMaster {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682 2024-11-16T09:58:00,103 DEBUG [master/037c58a19ddb:0:becomeActiveMaster {}] regionserver.HRegion(1048): stopping wal replay for 1595e783b53d99cd5eef43b6debb2682 2024-11-16T09:58:00,103 DEBUG [master/037c58a19ddb:0:becomeActiveMaster {}] regionserver.HRegion(1060): Cleaning up temporary data for 1595e783b53d99cd5eef43b6debb2682 2024-11-16T09:58:00,106 DEBUG [master/037c58a19ddb:0:becomeActiveMaster {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table master:store descriptor;using region.getMemStoreFlushHeapSize/# of families (32.0 M)) instead. 2024-11-16T09:58:00,110 DEBUG [master/037c58a19ddb:0:becomeActiveMaster {}] regionserver.HRegion(1093): writing seq id for 1595e783b53d99cd5eef43b6debb2682 2024-11-16T09:58:00,114 DEBUG [master/037c58a19ddb:0:becomeActiveMaster {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-16T09:58:00,115 INFO [master/037c58a19ddb:0:becomeActiveMaster {}] regionserver.HRegion(1114): Opened 1595e783b53d99cd5eef43b6debb2682; next sequenceid=2; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=69438980, jitterRate=0.034721434116363525}, FlushLargeStoresPolicy{flushSizeLowerBound=33554432} 2024-11-16T09:58:00,120 DEBUG [master/037c58a19ddb:0:becomeActiveMaster {}] regionserver.HRegion(1006): Region open journal for 1595e783b53d99cd5eef43b6debb2682: Writing region info on filesystem at 1731751080015Initializing all the Stores at 1731751080017 (+2 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731751080018 (+1 ms)Instantiating store for column family {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731751080018Instantiating store for column family {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731751080019 (+1 ms)Instantiating store for column family {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731751080019Cleaning up temporary data from old regions at 1731751080103 (+84 ms)Region opened successfully at 1731751080120 (+17 ms) 2024-11-16T09:58:00,122 INFO [master/037c58a19ddb:0:becomeActiveMaster {}] region.MasterRegionFlusherAndCompactor(122): Constructor flushSize=134217728, flushPerChanges=1000000, flushIntervalMs=900000, compactMin=4 2024-11-16T09:58:00,155 DEBUG [master/037c58a19ddb:0:becomeActiveMaster {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@77d8e150, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=037c58a19ddb/172.17.0.2:0 2024-11-16T09:58:00,187 INFO [master/037c58a19ddb:0:becomeActiveMaster {}] master.HMaster(912): No meta location available on zookeeper, skip migrating... 2024-11-16T09:58:00,199 INFO [master/037c58a19ddb:0:becomeActiveMaster {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=5 2024-11-16T09:58:00,199 INFO [master/037c58a19ddb:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(626): Starting 5 core workers (bigger of cpus/4 or 16) with max (burst) worker count=50 2024-11-16T09:58:00,202 INFO [master/037c58a19ddb:0:becomeActiveMaster {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery... 2024-11-16T09:58:00,204 INFO [master/037c58a19ddb:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 1 msec 2024-11-16T09:58:00,209 INFO [master/037c58a19ddb:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 4 msec 2024-11-16T09:58:00,209 INFO [master/037c58a19ddb:0:becomeActiveMaster {}] procedure2.RemoteProcedureDispatcher(96): Instantiated, coreThreads=3 (allowCoreThreadTimeOut=true), queueMaxSize=32, operationDelay=150 2024-11-16T09:58:00,234 INFO [master/037c58a19ddb:0:becomeActiveMaster {}] master.RegionServerTracker(127): Upgrading RegionServerTracker to active master mode; 0 have existingServerCrashProcedures, 0 possibly 'live' servers, and 0 'splitting'. 2024-11-16T09:58:00,243 DEBUG [master/037c58a19ddb:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:37877-0x101b083d7320000, quorum=127.0.0.1:49647, baseZNode=/hbase Unable to get data of znode /hbase/balancer because node does not exist (not necessarily an error) 2024-11-16T09:58:00,245 DEBUG [master/037c58a19ddb:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/balancer already deleted, retry=false 2024-11-16T09:58:00,247 INFO [master/037c58a19ddb:0:becomeActiveMaster {}] normalizer.SimpleRegionNormalizer(163): Updated configuration for key 'hbase.normalizer.merge.min_region_size.mb' from 0 to 1 2024-11-16T09:58:00,249 DEBUG [master/037c58a19ddb:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:37877-0x101b083d7320000, quorum=127.0.0.1:49647, baseZNode=/hbase Unable to get data of znode /hbase/normalizer because node does not exist (not necessarily an error) 2024-11-16T09:58:00,250 DEBUG [master/037c58a19ddb:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/normalizer already deleted, retry=false 2024-11-16T09:58:00,252 INFO [master/037c58a19ddb:0:becomeActiveMaster {}] normalizer.RegionNormalizerWorker(137): Normalizer rate limit set to unlimited 2024-11-16T09:58:00,256 DEBUG [master/037c58a19ddb:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:37877-0x101b083d7320000, quorum=127.0.0.1:49647, baseZNode=/hbase Unable to get data of znode /hbase/switch/split because node does not exist (not necessarily an error) 2024-11-16T09:58:00,257 DEBUG [master/037c58a19ddb:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/switch/split already deleted, retry=false 2024-11-16T09:58:00,258 DEBUG [master/037c58a19ddb:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:37877-0x101b083d7320000, quorum=127.0.0.1:49647, baseZNode=/hbase Unable to get data of znode /hbase/switch/merge because node does not exist (not necessarily an error) 2024-11-16T09:58:00,260 DEBUG [master/037c58a19ddb:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/switch/merge already deleted, retry=false 2024-11-16T09:58:00,277 DEBUG [master/037c58a19ddb:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:37877-0x101b083d7320000, quorum=127.0.0.1:49647, baseZNode=/hbase Unable to get data of znode /hbase/snapshot-cleanup because node does not exist (not necessarily an error) 2024-11-16T09:58:00,279 DEBUG [master/037c58a19ddb:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/snapshot-cleanup already deleted, retry=false 2024-11-16T09:58:00,282 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:37267-0x101b083d7320001, quorum=127.0.0.1:49647, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running 2024-11-16T09:58:00,282 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:37877-0x101b083d7320000, quorum=127.0.0.1:49647, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running 2024-11-16T09:58:00,283 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:37267-0x101b083d7320001, quorum=127.0.0.1:49647, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-16T09:58:00,283 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:37877-0x101b083d7320000, quorum=127.0.0.1:49647, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-16T09:58:00,285 INFO [master/037c58a19ddb:0:becomeActiveMaster {}] master.HMaster(856): Active/primary master=037c58a19ddb,37877,1731751077131, sessionid=0x101b083d7320000, setting cluster-up flag (Was=false) 2024-11-16T09:58:00,298 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:37267-0x101b083d7320001, quorum=127.0.0.1:49647, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-16T09:58:00,298 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:37877-0x101b083d7320000, quorum=127.0.0.1:49647, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-16T09:58:00,304 DEBUG [master/037c58a19ddb:0:becomeActiveMaster {}] procedure.ZKProcedureUtil(251): Clearing all znodes /hbase/flush-table-proc/acquired, /hbase/flush-table-proc/reached, /hbase/flush-table-proc/abort 2024-11-16T09:58:00,305 DEBUG [master/037c58a19ddb:0:becomeActiveMaster {}] procedure.ZKProcedureCoordinator(245): Starting controller for procedure member=037c58a19ddb,37877,1731751077131 2024-11-16T09:58:00,311 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:37267-0x101b083d7320001, quorum=127.0.0.1:49647, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-16T09:58:00,311 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:37877-0x101b083d7320000, quorum=127.0.0.1:49647, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-16T09:58:00,316 DEBUG [master/037c58a19ddb:0:becomeActiveMaster {}] procedure.ZKProcedureUtil(251): Clearing all znodes /hbase/online-snapshot/acquired, /hbase/online-snapshot/reached, /hbase/online-snapshot/abort 2024-11-16T09:58:00,318 DEBUG [master/037c58a19ddb:0:becomeActiveMaster {}] procedure.ZKProcedureCoordinator(245): Starting controller for procedure member=037c58a19ddb,37877,1731751077131 2024-11-16T09:58:00,324 INFO [master/037c58a19ddb:0:becomeActiveMaster {}] master.ServerManager(1185): No .lastflushedseqids found at hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/.lastflushedseqids will record last flushed sequence id for regions by regionserver report all over again 2024-11-16T09:58:00,347 INFO [RS:0;037c58a19ddb:37267 {}] regionserver.HRegionServer(746): ClusterId : 1fe46ee8-cb44-44b1-8251-37103629f786 2024-11-16T09:58:00,350 DEBUG [RS:0;037c58a19ddb:37267 {}] procedure.RegionServerProcedureManagerHost(43): Procedure flush-table-proc initializing 2024-11-16T09:58:00,356 DEBUG [RS:0;037c58a19ddb:37267 {}] procedure.RegionServerProcedureManagerHost(45): Procedure flush-table-proc initialized 2024-11-16T09:58:00,356 DEBUG [RS:0;037c58a19ddb:37267 {}] procedure.RegionServerProcedureManagerHost(43): Procedure online-snapshot initializing 2024-11-16T09:58:00,359 DEBUG [RS:0;037c58a19ddb:37267 {}] procedure.RegionServerProcedureManagerHost(45): Procedure online-snapshot initialized 2024-11-16T09:58:00,359 DEBUG [RS:0;037c58a19ddb:37267 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@735ef924, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=037c58a19ddb/172.17.0.2:0 2024-11-16T09:58:00,379 DEBUG [RS:0;037c58a19ddb:37267 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:RS:0;037c58a19ddb:37267 2024-11-16T09:58:00,384 INFO [RS:0;037c58a19ddb:37267 {}] regionserver.RegionServerCoprocessorHost(66): System coprocessor loading is enabled 2024-11-16T09:58:00,384 INFO [RS:0;037c58a19ddb:37267 {}] regionserver.RegionServerCoprocessorHost(67): Table coprocessor loading is enabled 2024-11-16T09:58:00,384 DEBUG [RS:0;037c58a19ddb:37267 {}] regionserver.HRegionServer(832): About to register with Master. 2024-11-16T09:58:00,387 INFO [RS:0;037c58a19ddb:37267 {}] regionserver.HRegionServer(2659): reportForDuty to master=037c58a19ddb,37877,1731751077131 with port=37267, startcode=1731751077844 2024-11-16T09:58:00,401 DEBUG [RS:0;037c58a19ddb:37267 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=RegionServerStatusService, sasl=false 2024-11-16T09:58:00,404 DEBUG [master/037c58a19ddb:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(1139): Stored pid=1, state=RUNNABLE:INIT_META_WRITE_FS_LAYOUT, hasLock=false; InitMetaProcedure table=hbase:meta 2024-11-16T09:58:00,415 INFO [master/037c58a19ddb:0:becomeActiveMaster {}] balancer.BaseLoadBalancer(416): slop=0.2 2024-11-16T09:58:00,423 INFO [master/037c58a19ddb:0:becomeActiveMaster {}] balancer.StochasticLoadBalancer(272): Loaded config; maxSteps=1000000, runMaxSteps=false, stepsPerRegion=800, maxRunningTime=30000, isByTable=false, CostFunctions=[RegionCountSkewCostFunction, PrimaryRegionCountSkewCostFunction, MoveCostFunction, ServerLocalityCostFunction, RackLocalityCostFunction, TableSkewCostFunction, RegionReplicaHostCostFunction, RegionReplicaRackCostFunction, ReadRequestCostFunction, CPRequestCostFunction, WriteRequestCostFunction, MemStoreSizeCostFunction, StoreFileCostFunction] , sum of multiplier of cost functions = 0.0 etc. 2024-11-16T09:58:00,430 DEBUG [master/037c58a19ddb:0:becomeActiveMaster {}] balancer.RegionHDFSBlockLocationFinder(133): Skipping locality-based refresh due to oldStatus=null, newStatus=Master: 037c58a19ddb,37877,1731751077131 Number of backup masters: 0 Number of live region servers: 0 Number of dead region servers: 0 Number of unknown region servers: 0 Average load: 0.0 Number of requests: 0 Number of regions: 0 Number of regions in transition: 0 2024-11-16T09:58:00,439 DEBUG [master/037c58a19ddb:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_OPEN_REGION-master/037c58a19ddb:0, corePoolSize=5, maxPoolSize=5 2024-11-16T09:58:00,439 DEBUG [master/037c58a19ddb:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_CLOSE_REGION-master/037c58a19ddb:0, corePoolSize=5, maxPoolSize=5 2024-11-16T09:58:00,439 DEBUG [master/037c58a19ddb:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_SERVER_OPERATIONS-master/037c58a19ddb:0, corePoolSize=5, maxPoolSize=5 2024-11-16T09:58:00,440 DEBUG [master/037c58a19ddb:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_META_SERVER_OPERATIONS-master/037c58a19ddb:0, corePoolSize=5, maxPoolSize=5 2024-11-16T09:58:00,440 DEBUG [master/037c58a19ddb:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=M_LOG_REPLAY_OPS-master/037c58a19ddb:0, corePoolSize=10, maxPoolSize=10 2024-11-16T09:58:00,440 DEBUG [master/037c58a19ddb:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_SNAPSHOT_OPERATIONS-master/037c58a19ddb:0, corePoolSize=1, maxPoolSize=1 2024-11-16T09:58:00,441 DEBUG [master/037c58a19ddb:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_MERGE_OPERATIONS-master/037c58a19ddb:0, corePoolSize=2, maxPoolSize=2 2024-11-16T09:58:00,441 DEBUG [master/037c58a19ddb:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_TABLE_OPERATIONS-master/037c58a19ddb:0, corePoolSize=1, maxPoolSize=1 2024-11-16T09:58:00,443 INFO [master/037c58a19ddb:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1731751110443 2024-11-16T09:58:00,443 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:37309, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins.hfs.0 (auth:SIMPLE), service=RegionServerStatusService 2024-11-16T09:58:00,445 INFO [master/037c58a19ddb:0:becomeActiveMaster {}] cleaner.DirScanPool(74): log_cleaner Cleaner pool size is 1 2024-11-16T09:58:00,446 INFO [master/037c58a19ddb:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveLogCleaner 2024-11-16T09:58:00,446 DEBUG [PEWorker-1 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_WRITE_FS_LAYOUT, hasLock=true; InitMetaProcedure table=hbase:meta 2024-11-16T09:58:00,447 INFO [PEWorker-1 {}] procedure.InitMetaProcedure(76): BOOTSTRAP: creating hbase:meta region 2024-11-16T09:58:00,449 DEBUG [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=37877 {}] ipc.MetricsHBaseServer(152): Unknown exception type org.apache.hadoop.hbase.ipc.ServerNotRunningYetException: Server is not running yet at org.apache.hadoop.hbase.master.HMaster.checkServiceStarted(HMaster.java:3334) ~[classes/:?] at org.apache.hadoop.hbase.master.MasterRpcServices.regionServerStartup(MasterRpcServices.java:667) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16714) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T09:58:00,450 INFO [master/037c58a19ddb:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.replication.master.ReplicationLogCleaner 2024-11-16T09:58:00,450 INFO [master/037c58a19ddb:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveMasterLocalStoreWALCleaner 2024-11-16T09:58:00,451 INFO [master/037c58a19ddb:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveProcedureWALCleaner 2024-11-16T09:58:00,451 INFO [master/037c58a19ddb:0:becomeActiveMaster {}] cleaner.LogCleaner(148): Creating 1 old WALs cleaner threads 2024-11-16T09:58:00,452 INFO [master/037c58a19ddb:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=LogsCleaner, period=600000, unit=MILLISECONDS is enabled. 2024-11-16T09:58:00,455 DEBUG [PEWorker-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T09:58:00,455 INFO [PEWorker-1 {}] util.FSTableDescriptors(156): Creating new hbase:meta table descriptor 'hbase:meta', {TABLE_ATTRIBUTES => {IS_META => 'true', coprocessor$1 => '|org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint|536870911|', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} 2024-11-16T09:58:00,457 INFO [master/037c58a19ddb:0:becomeActiveMaster {}] cleaner.DirScanPool(74): hfile_cleaner Cleaner pool size is 2 2024-11-16T09:58:00,458 INFO [master/037c58a19ddb:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveMasterLocalStoreHFileCleaner 2024-11-16T09:58:00,458 INFO [master/037c58a19ddb:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.HFileLinkCleaner 2024-11-16T09:58:00,461 INFO [master/037c58a19ddb:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.snapshot.SnapshotHFileCleaner 2024-11-16T09:58:00,461 INFO [master/037c58a19ddb:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveHFileCleaner 2024-11-16T09:58:00,467 DEBUG [master/037c58a19ddb:0:becomeActiveMaster {}] cleaner.HFileCleaner(260): Starting for large file=Thread[master/037c58a19ddb:0:becomeActiveMaster-HFileCleaner.large.0-1731751080462,5,FailOnTimeoutGroup] 2024-11-16T09:58:00,468 DEBUG [master/037c58a19ddb:0:becomeActiveMaster {}] cleaner.HFileCleaner(275): Starting for small files=Thread[master/037c58a19ddb:0:becomeActiveMaster-HFileCleaner.small.0-1731751080467,5,FailOnTimeoutGroup] 2024-11-16T09:58:00,468 INFO [master/037c58a19ddb:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=HFileCleaner, period=600000, unit=MILLISECONDS is enabled. 2024-11-16T09:58:00,468 INFO [master/037c58a19ddb:0:becomeActiveMaster {}] master.HMaster(1741): Reopening regions with very high storeFileRefCount is disabled. Provide threshold value > 0 for hbase.regions.recovery.store.file.ref.count to enable it. 2024-11-16T09:58:00,470 INFO [master/037c58a19ddb:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationBarrierCleaner, period=43200000, unit=MILLISECONDS is enabled. 2024-11-16T09:58:00,470 INFO [master/037c58a19ddb:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=SnapshotCleaner, period=1800000, unit=MILLISECONDS is enabled. 2024-11-16T09:58:00,471 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37591 is added to blk_1073741831_1007 (size=1321) 2024-11-16T09:58:00,478 DEBUG [RS:0;037c58a19ddb:37267 {}] regionserver.HRegionServer(2683): Master is not running yet 2024-11-16T09:58:00,478 WARN [RS:0;037c58a19ddb:37267 {}] regionserver.HRegionServer(841): reportForDuty failed; sleeping 100 ms and then retrying. 2024-11-16T09:58:00,579 INFO [RS:0;037c58a19ddb:37267 {}] regionserver.HRegionServer(2659): reportForDuty to master=037c58a19ddb,37877,1731751077131 with port=37267, startcode=1731751077844 2024-11-16T09:58:00,582 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=37877 {}] master.ServerManager(363): Checking decommissioned status of RegionServer 037c58a19ddb,37267,1731751077844 2024-11-16T09:58:00,584 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=37877 {}] master.ServerManager(517): Registering regionserver=037c58a19ddb,37267,1731751077844 2024-11-16T09:58:00,593 DEBUG [RS:0;037c58a19ddb:37267 {}] regionserver.HRegionServer(1440): Config from master: hbase.rootdir=hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592 2024-11-16T09:58:00,593 DEBUG [RS:0;037c58a19ddb:37267 {}] regionserver.HRegionServer(1440): Config from master: fs.defaultFS=hdfs://localhost:44755 2024-11-16T09:58:00,594 DEBUG [RS:0;037c58a19ddb:37267 {}] regionserver.HRegionServer(1440): Config from master: hbase.master.info.port=-1 2024-11-16T09:58:00,598 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:37877-0x101b083d7320000, quorum=127.0.0.1:49647, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-11-16T09:58:00,599 DEBUG [RS:0;037c58a19ddb:37267 {}] zookeeper.ZKUtil(111): regionserver:37267-0x101b083d7320001, quorum=127.0.0.1:49647, baseZNode=/hbase Set watcher on existing znode=/hbase/rs/037c58a19ddb,37267,1731751077844 2024-11-16T09:58:00,599 WARN [RS:0;037c58a19ddb:37267 {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-11-16T09:58:00,599 INFO [RS:0;037c58a19ddb:37267 {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-16T09:58:00,599 DEBUG [RS:0;037c58a19ddb:37267 {}] regionserver.HRegionServer(1793): logDir=hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/WALs/037c58a19ddb,37267,1731751077844 2024-11-16T09:58:00,601 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(179): RegionServer ephemeral node created, adding [037c58a19ddb,37267,1731751077844] 2024-11-16T09:58:00,625 INFO [RS:0;037c58a19ddb:37267 {}] regionserver.MetricsRegionServerWrapperImpl(120): Computing regionserver metrics every 5000 milliseconds 2024-11-16T09:58:00,638 INFO [RS:0;037c58a19ddb:37267 {}] regionserver.MemStoreFlusher(131): globalMemStoreLimit=880 M, globalMemStoreLimitLowMark=836 M, Offheap=false 2024-11-16T09:58:00,643 INFO [RS:0;037c58a19ddb:37267 {}] throttle.PressureAwareCompactionThroughputController(131): Compaction throughput configurations, higher bound: 100.00 MB/second, lower bound 50.00 MB/second, off peak: unlimited, tuning period: 60000 ms 2024-11-16T09:58:00,643 INFO [RS:0;037c58a19ddb:37267 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS is enabled. 2024-11-16T09:58:00,644 INFO [RS:0;037c58a19ddb:37267 {}] regionserver.HRegionServer$CompactionChecker(1680): CompactionChecker runs every PT1S 2024-11-16T09:58:00,650 INFO [RS:0;037c58a19ddb:37267 {}] hbase.ExecutorStatusChore(48): ExecutorStatusChore runs every 1mins, 0sec 2024-11-16T09:58:00,651 INFO [RS:0;037c58a19ddb:37267 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactedHFilesCleaner, period=120000, unit=MILLISECONDS is enabled. 2024-11-16T09:58:00,651 DEBUG [RS:0;037c58a19ddb:37267 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_REGION-regionserver/037c58a19ddb:0, corePoolSize=1, maxPoolSize=1 2024-11-16T09:58:00,651 DEBUG [RS:0;037c58a19ddb:37267 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_META-regionserver/037c58a19ddb:0, corePoolSize=1, maxPoolSize=1 2024-11-16T09:58:00,652 DEBUG [RS:0;037c58a19ddb:37267 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_PRIORITY_REGION-regionserver/037c58a19ddb:0, corePoolSize=1, maxPoolSize=1 2024-11-16T09:58:00,652 DEBUG [RS:0;037c58a19ddb:37267 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_REGION-regionserver/037c58a19ddb:0, corePoolSize=1, maxPoolSize=1 2024-11-16T09:58:00,652 DEBUG [RS:0;037c58a19ddb:37267 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_META-regionserver/037c58a19ddb:0, corePoolSize=1, maxPoolSize=1 2024-11-16T09:58:00,652 DEBUG [RS:0;037c58a19ddb:37267 {}] executor.ExecutorService(95): Starting executor service name=RS_LOG_REPLAY_OPS-regionserver/037c58a19ddb:0, corePoolSize=2, maxPoolSize=2 2024-11-16T09:58:00,652 DEBUG [RS:0;037c58a19ddb:37267 {}] executor.ExecutorService(95): Starting executor service name=RS_COMPACTED_FILES_DISCHARGER-regionserver/037c58a19ddb:0, corePoolSize=1, maxPoolSize=1 2024-11-16T09:58:00,652 DEBUG [RS:0;037c58a19ddb:37267 {}] executor.ExecutorService(95): Starting executor service name=RS_REGION_REPLICA_FLUSH_OPS-regionserver/037c58a19ddb:0, corePoolSize=1, maxPoolSize=1 2024-11-16T09:58:00,653 DEBUG [RS:0;037c58a19ddb:37267 {}] executor.ExecutorService(95): Starting executor service name=RS_REFRESH_PEER-regionserver/037c58a19ddb:0, corePoolSize=1, maxPoolSize=1 2024-11-16T09:58:00,653 DEBUG [RS:0;037c58a19ddb:37267 {}] executor.ExecutorService(95): Starting executor service name=RS_REPLAY_SYNC_REPLICATION_WAL-regionserver/037c58a19ddb:0, corePoolSize=1, maxPoolSize=1 2024-11-16T09:58:00,653 DEBUG [RS:0;037c58a19ddb:37267 {}] executor.ExecutorService(95): Starting executor service name=RS_SWITCH_RPC_THROTTLE-regionserver/037c58a19ddb:0, corePoolSize=1, maxPoolSize=1 2024-11-16T09:58:00,653 DEBUG [RS:0;037c58a19ddb:37267 {}] executor.ExecutorService(95): Starting executor service name=RS_CLAIM_REPLICATION_QUEUE-regionserver/037c58a19ddb:0, corePoolSize=1, maxPoolSize=1 2024-11-16T09:58:00,653 DEBUG [RS:0;037c58a19ddb:37267 {}] executor.ExecutorService(95): Starting executor service name=RS_SNAPSHOT_OPERATIONS-regionserver/037c58a19ddb:0, corePoolSize=3, maxPoolSize=3 2024-11-16T09:58:00,653 DEBUG [RS:0;037c58a19ddb:37267 {}] executor.ExecutorService(95): Starting executor service name=RS_FLUSH_OPERATIONS-regionserver/037c58a19ddb:0, corePoolSize=3, maxPoolSize=3 2024-11-16T09:58:00,654 INFO [RS:0;037c58a19ddb:37267 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionChecker, period=1000, unit=MILLISECONDS is enabled. 2024-11-16T09:58:00,654 INFO [RS:0;037c58a19ddb:37267 {}] hbase.ChoreService(168): Chore ScheduledChore name=MemstoreFlusherChore, period=1000, unit=MILLISECONDS is enabled. 2024-11-16T09:58:00,654 INFO [RS:0;037c58a19ddb:37267 {}] hbase.ChoreService(168): Chore ScheduledChore name=ExecutorStatusChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-16T09:58:00,655 INFO [RS:0;037c58a19ddb:37267 {}] hbase.ChoreService(168): Chore ScheduledChore name=nonceCleaner, period=360000, unit=MILLISECONDS is enabled. 2024-11-16T09:58:00,655 INFO [RS:0;037c58a19ddb:37267 {}] hbase.ChoreService(168): Chore ScheduledChore name=BrokenStoreFileCleaner, period=21600000, unit=MILLISECONDS is enabled. 2024-11-16T09:58:00,655 INFO [RS:0;037c58a19ddb:37267 {}] hbase.ChoreService(168): Chore ScheduledChore name=037c58a19ddb,37267,1731751077844-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-11-16T09:58:00,674 INFO [RS:0;037c58a19ddb:37267 {}] regionserver.HeapMemoryManager(209): Starting, tuneOn=false 2024-11-16T09:58:00,676 INFO [RS:0;037c58a19ddb:37267 {}] hbase.ChoreService(168): Chore ScheduledChore name=037c58a19ddb,37267,1731751077844-HeapMemoryTunerChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-16T09:58:00,677 INFO [RS:0;037c58a19ddb:37267 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-11-16T09:58:00,677 INFO [RS:0;037c58a19ddb:37267 {}] regionserver.Replication(171): 037c58a19ddb,37267,1731751077844 started 2024-11-16T09:58:00,695 INFO [RS:0;037c58a19ddb:37267 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-11-16T09:58:00,696 INFO [RS:0;037c58a19ddb:37267 {}] regionserver.HRegionServer(1482): Serving as 037c58a19ddb,37267,1731751077844, RpcServer on 037c58a19ddb/172.17.0.2:37267, sessionid=0x101b083d7320001 2024-11-16T09:58:00,697 DEBUG [RS:0;037c58a19ddb:37267 {}] procedure.RegionServerProcedureManagerHost(51): Procedure flush-table-proc starting 2024-11-16T09:58:00,697 DEBUG [RS:0;037c58a19ddb:37267 {}] flush.RegionServerFlushTableProcedureManager(108): Start region server flush procedure manager 037c58a19ddb,37267,1731751077844 2024-11-16T09:58:00,697 DEBUG [RS:0;037c58a19ddb:37267 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member '037c58a19ddb,37267,1731751077844' 2024-11-16T09:58:00,697 DEBUG [RS:0;037c58a19ddb:37267 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/flush-table-proc/abort' 2024-11-16T09:58:00,698 DEBUG [RS:0;037c58a19ddb:37267 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/flush-table-proc/acquired' 2024-11-16T09:58:00,699 DEBUG [RS:0;037c58a19ddb:37267 {}] procedure.RegionServerProcedureManagerHost(53): Procedure flush-table-proc started 2024-11-16T09:58:00,699 DEBUG [RS:0;037c58a19ddb:37267 {}] procedure.RegionServerProcedureManagerHost(51): Procedure online-snapshot starting 2024-11-16T09:58:00,699 DEBUG [RS:0;037c58a19ddb:37267 {}] snapshot.RegionServerSnapshotManager(126): Start Snapshot Manager 037c58a19ddb,37267,1731751077844 2024-11-16T09:58:00,699 DEBUG [RS:0;037c58a19ddb:37267 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member '037c58a19ddb,37267,1731751077844' 2024-11-16T09:58:00,699 DEBUG [RS:0;037c58a19ddb:37267 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/online-snapshot/abort' 2024-11-16T09:58:00,700 DEBUG [RS:0;037c58a19ddb:37267 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/online-snapshot/acquired' 2024-11-16T09:58:00,700 DEBUG [RS:0;037c58a19ddb:37267 {}] procedure.RegionServerProcedureManagerHost(53): Procedure online-snapshot started 2024-11-16T09:58:00,701 INFO [RS:0;037c58a19ddb:37267 {}] quotas.RegionServerRpcQuotaManager(64): Quota support disabled 2024-11-16T09:58:00,701 INFO [RS:0;037c58a19ddb:37267 {}] quotas.RegionServerSpaceQuotaManager(80): Quota support disabled, not starting space quota manager. 2024-11-16T09:58:00,806 INFO [RS:0;037c58a19ddb:37267 {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-11-16T09:58:00,810 INFO [RS:0;037c58a19ddb:37267 {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=037c58a19ddb%2C37267%2C1731751077844, suffix=, logDir=hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/WALs/037c58a19ddb,37267,1731751077844, archiveDir=hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/oldWALs, maxLogs=32 2024-11-16T09:58:00,828 DEBUG [RS:0;037c58a19ddb:37267 {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/WALs/037c58a19ddb,37267,1731751077844/037c58a19ddb%2C37267%2C1731751077844.1731751080812, exclude list is [], retry=0 2024-11-16T09:58:00,834 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:37591,DS-93631843-23e1-4c59-bec2-65fdae3c8e7b,DISK] 2024-11-16T09:58:00,838 INFO [RS:0;037c58a19ddb:37267 {}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/WALs/037c58a19ddb,37267,1731751077844/037c58a19ddb%2C37267%2C1731751077844.1731751080812 2024-11-16T09:58:00,839 DEBUG [RS:0;037c58a19ddb:37267 {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:36709:36709)] 2024-11-16T09:58:00,874 INFO [PEWorker-1 {}] util.FSTableDescriptors(163): Updated hbase:meta table descriptor to hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/hbase/meta/.tabledesc/.tableinfo.0000000001.1321 2024-11-16T09:58:00,875 INFO [PEWorker-1 {}] regionserver.HRegion(7572): creating {ENCODED => 1588230740, NAME => 'hbase:meta,,1', STARTKEY => '', ENDKEY => ''}, tableDescriptor='hbase:meta', {TABLE_ATTRIBUTES => {IS_META => 'true', coprocessor$1 => '|org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint|536870911|', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, regionDir=hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592 2024-11-16T09:58:00,884 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37591 is added to blk_1073741833_1009 (size=32) 2024-11-16T09:58:01,287 DEBUG [PEWorker-1 {}] regionserver.HRegion(898): Instantiated hbase:meta,,1.1588230740; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-16T09:58:01,290 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1588230740 2024-11-16T09:58:01,293 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName info 2024-11-16T09:58:01,293 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T09:58:01,294 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-16T09:58:01,294 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family ns of region 1588230740 2024-11-16T09:58:01,296 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName ns 2024-11-16T09:58:01,297 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T09:58:01,297 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/ns, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-16T09:58:01,298 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rep_barrier of region 1588230740 2024-11-16T09:58:01,300 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName rep_barrier 2024-11-16T09:58:01,300 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T09:58:01,301 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/rep_barrier, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-16T09:58:01,301 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family table of region 1588230740 2024-11-16T09:58:01,303 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName table 2024-11-16T09:58:01,303 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T09:58:01,304 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/table, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-16T09:58:01,304 DEBUG [PEWorker-1 {}] regionserver.HRegion(1038): replaying wal for 1588230740 2024-11-16T09:58:01,306 DEBUG [PEWorker-1 {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/hbase/meta/1588230740 2024-11-16T09:58:01,306 DEBUG [PEWorker-1 {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/hbase/meta/1588230740 2024-11-16T09:58:01,309 DEBUG [PEWorker-1 {}] regionserver.HRegion(1048): stopping wal replay for 1588230740 2024-11-16T09:58:01,309 DEBUG [PEWorker-1 {}] regionserver.HRegion(1060): Cleaning up temporary data for 1588230740 2024-11-16T09:58:01,310 DEBUG [PEWorker-1 {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table hbase:meta descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-11-16T09:58:01,312 DEBUG [PEWorker-1 {}] regionserver.HRegion(1093): writing seq id for 1588230740 2024-11-16T09:58:01,316 DEBUG [PEWorker-1 {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/hbase/meta/1588230740/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-16T09:58:01,317 INFO [PEWorker-1 {}] regionserver.HRegion(1114): Opened 1588230740; next sequenceid=2; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=63132746, jitterRate=-0.059248775243759155}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-11-16T09:58:01,319 DEBUG [PEWorker-1 {}] regionserver.HRegion(1006): Region open journal for 1588230740: Writing region info on filesystem at 1731751081288Initializing all the Stores at 1731751081290 (+2 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731751081290Instantiating store for column family {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731751081290Instantiating store for column family {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731751081290Instantiating store for column family {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731751081290Cleaning up temporary data from old regions at 1731751081309 (+19 ms)Region opened successfully at 1731751081319 (+10 ms) 2024-11-16T09:58:01,319 DEBUG [PEWorker-1 {}] regionserver.HRegion(1722): Closing 1588230740, disabling compactions & flushes 2024-11-16T09:58:01,319 INFO [PEWorker-1 {}] regionserver.HRegion(1755): Closing region hbase:meta,,1.1588230740 2024-11-16T09:58:01,319 DEBUG [PEWorker-1 {}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:meta,,1.1588230740 2024-11-16T09:58:01,319 DEBUG [PEWorker-1 {}] regionserver.HRegion(1843): Acquired close lock on hbase:meta,,1.1588230740 after waiting 0 ms 2024-11-16T09:58:01,319 DEBUG [PEWorker-1 {}] regionserver.HRegion(1853): Updates disabled for region hbase:meta,,1.1588230740 2024-11-16T09:58:01,320 INFO [PEWorker-1 {}] regionserver.HRegion(1973): Closed hbase:meta,,1.1588230740 2024-11-16T09:58:01,321 DEBUG [PEWorker-1 {}] regionserver.HRegion(1676): Region close journal for 1588230740: Waiting for close lock at 1731751081319Disabling compacts and flushes for region at 1731751081319Disabling writes for close at 1731751081319Writing region close event to WAL at 1731751081320 (+1 ms)Closed at 1731751081320 2024-11-16T09:58:01,324 DEBUG [PEWorker-1 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_ASSIGN_META, hasLock=true; InitMetaProcedure table=hbase:meta 2024-11-16T09:58:01,324 INFO [PEWorker-1 {}] procedure.InitMetaProcedure(108): Going to assign meta 2024-11-16T09:58:01,331 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN}] 2024-11-16T09:58:01,339 INFO [PEWorker-2 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-16T09:58:01,342 INFO [PEWorker-2 {}] assignment.TransitRegionStateProcedure(269): Starting pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN; state=OFFLINE, location=null; forceNewPlan=false, retain=false 2024-11-16T09:58:01,495 DEBUG [037c58a19ddb:37877 {}] assignment.AssignmentManager(2472): Processing assignQueue; systemServersCount=1, allServersCount=1 2024-11-16T09:58:01,508 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=2 updating hbase:meta row=1588230740, regionState=OPENING, regionLocation=037c58a19ddb,37267,1731751077844 2024-11-16T09:58:01,515 INFO [PEWorker-3 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 037c58a19ddb,37267,1731751077844, state=OPENING 2024-11-16T09:58:01,520 DEBUG [PEWorker-3 {}] zookeeper.MetaTableLocator(183): hbase:meta region location doesn't exist, create it 2024-11-16T09:58:01,522 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:37267-0x101b083d7320001, quorum=127.0.0.1:49647, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-16T09:58:01,522 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:37877-0x101b083d7320000, quorum=127.0.0.1:49647, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-16T09:58:01,523 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-16T09:58:01,523 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-16T09:58:01,525 DEBUG [PEWorker-3 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-16T09:58:01,527 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=3, ppid=2, state=RUNNABLE, hasLock=false; OpenRegionProcedure 1588230740, server=037c58a19ddb,37267,1731751077844}] 2024-11-16T09:58:01,702 DEBUG [RSProcedureDispatcher-pool-0 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=AdminService, sasl=false 2024-11-16T09:58:01,706 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:40369, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=AdminService 2024-11-16T09:58:01,716 INFO [RS_OPEN_META-regionserver/037c58a19ddb:0-0 {event_type=M_RS_OPEN_META, pid=3}] handler.AssignRegionHandler(132): Open hbase:meta,,1.1588230740 2024-11-16T09:58:01,716 INFO [RS_OPEN_META-regionserver/037c58a19ddb:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-16T09:58:01,717 INFO [RS_OPEN_META-regionserver/037c58a19ddb:0-0 {event_type=M_RS_OPEN_META, pid=3}] monitor.StreamSlowMonitor(122): New stream slow monitor .meta 2024-11-16T09:58:01,720 INFO [RS_OPEN_META-regionserver/037c58a19ddb:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=037c58a19ddb%2C37267%2C1731751077844.meta, suffix=.meta, logDir=hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/WALs/037c58a19ddb,37267,1731751077844, archiveDir=hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/oldWALs, maxLogs=32 2024-11-16T09:58:01,736 DEBUG [RS_OPEN_META-regionserver/037c58a19ddb:0-0 {event_type=M_RS_OPEN_META, pid=3}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/WALs/037c58a19ddb,37267,1731751077844/037c58a19ddb%2C37267%2C1731751077844.meta.1731751081722.meta, exclude list is [], retry=0 2024-11-16T09:58:01,740 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:37591,DS-93631843-23e1-4c59-bec2-65fdae3c8e7b,DISK] 2024-11-16T09:58:01,743 INFO [RS_OPEN_META-regionserver/037c58a19ddb:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/WALs/037c58a19ddb,37267,1731751077844/037c58a19ddb%2C37267%2C1731751077844.meta.1731751081722.meta 2024-11-16T09:58:01,744 DEBUG [RS_OPEN_META-regionserver/037c58a19ddb:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:36709:36709)] 2024-11-16T09:58:01,744 DEBUG [RS_OPEN_META-regionserver/037c58a19ddb:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7752): Opening region: {ENCODED => 1588230740, NAME => 'hbase:meta,,1', STARTKEY => '', ENDKEY => ''} 2024-11-16T09:58:01,746 DEBUG [RS_OPEN_META-regionserver/037c58a19ddb:0-0 {event_type=M_RS_OPEN_META, pid=3}] coprocessor.CoprocessorHost(215): Loading coprocessor class org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint with path null and priority 536870911 2024-11-16T09:58:01,748 DEBUG [RS_OPEN_META-regionserver/037c58a19ddb:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(8280): Registered coprocessor service: region=hbase:meta,,1 service=MultiRowMutationService 2024-11-16T09:58:01,753 INFO [RS_OPEN_META-regionserver/037c58a19ddb:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.RegionCoprocessorHost(434): Loaded coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint from HTD of hbase:meta successfully. 2024-11-16T09:58:01,757 DEBUG [RS_OPEN_META-regionserver/037c58a19ddb:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table meta 1588230740 2024-11-16T09:58:01,758 DEBUG [RS_OPEN_META-regionserver/037c58a19ddb:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(898): Instantiated hbase:meta,,1.1588230740; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-16T09:58:01,758 DEBUG [RS_OPEN_META-regionserver/037c58a19ddb:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7794): checking encryption for 1588230740 2024-11-16T09:58:01,758 DEBUG [RS_OPEN_META-regionserver/037c58a19ddb:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7797): checking classloading for 1588230740 2024-11-16T09:58:01,762 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1588230740 2024-11-16T09:58:01,763 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName info 2024-11-16T09:58:01,763 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T09:58:01,764 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-16T09:58:01,765 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family ns of region 1588230740 2024-11-16T09:58:01,766 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName ns 2024-11-16T09:58:01,766 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T09:58:01,767 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/ns, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-16T09:58:01,767 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rep_barrier of region 1588230740 2024-11-16T09:58:01,769 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName rep_barrier 2024-11-16T09:58:01,769 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T09:58:01,769 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/rep_barrier, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-16T09:58:01,770 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family table of region 1588230740 2024-11-16T09:58:01,771 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName table 2024-11-16T09:58:01,771 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T09:58:01,772 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/table, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-16T09:58:01,772 DEBUG [RS_OPEN_META-regionserver/037c58a19ddb:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1038): replaying wal for 1588230740 2024-11-16T09:58:01,774 DEBUG [RS_OPEN_META-regionserver/037c58a19ddb:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/hbase/meta/1588230740 2024-11-16T09:58:01,776 DEBUG [RS_OPEN_META-regionserver/037c58a19ddb:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/hbase/meta/1588230740 2024-11-16T09:58:01,778 DEBUG [RS_OPEN_META-regionserver/037c58a19ddb:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1048): stopping wal replay for 1588230740 2024-11-16T09:58:01,778 DEBUG [RS_OPEN_META-regionserver/037c58a19ddb:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1060): Cleaning up temporary data for 1588230740 2024-11-16T09:58:01,779 DEBUG [RS_OPEN_META-regionserver/037c58a19ddb:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table hbase:meta descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-11-16T09:58:01,781 DEBUG [RS_OPEN_META-regionserver/037c58a19ddb:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1093): writing seq id for 1588230740 2024-11-16T09:58:01,783 INFO [RS_OPEN_META-regionserver/037c58a19ddb:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1114): Opened 1588230740; next sequenceid=2; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=62502123, jitterRate=-0.06864579021930695}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-11-16T09:58:01,783 DEBUG [RS_OPEN_META-regionserver/037c58a19ddb:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 1588230740 2024-11-16T09:58:01,785 DEBUG [RS_OPEN_META-regionserver/037c58a19ddb:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1006): Region open journal for 1588230740: Running coprocessor pre-open hook at 1731751081759Writing region info on filesystem at 1731751081759Initializing all the Stores at 1731751081761 (+2 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731751081761Instantiating store for column family {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731751081761Instantiating store for column family {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731751081761Instantiating store for column family {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731751081761Cleaning up temporary data from old regions at 1731751081778 (+17 ms)Running coprocessor post-open hooks at 1731751081783 (+5 ms)Region opened successfully at 1731751081785 (+2 ms) 2024-11-16T09:58:01,793 INFO [RS_OPEN_META-regionserver/037c58a19ddb:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegionServer(2236): Post open deploy tasks for hbase:meta,,1.1588230740, pid=3, masterSystemTime=1731751081693 2024-11-16T09:58:01,804 DEBUG [RS_OPEN_META-regionserver/037c58a19ddb:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegionServer(2266): Finished post open deploy task for hbase:meta,,1.1588230740 2024-11-16T09:58:01,804 INFO [RS_OPEN_META-regionserver/037c58a19ddb:0-0 {event_type=M_RS_OPEN_META, pid=3}] handler.AssignRegionHandler(153): Opened hbase:meta,,1.1588230740 2024-11-16T09:58:01,806 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=2 updating hbase:meta row=1588230740, regionState=OPEN, openSeqNum=2, regionLocation=037c58a19ddb,37267,1731751077844 2024-11-16T09:58:01,808 INFO [PEWorker-5 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 037c58a19ddb,37267,1731751077844, state=OPEN 2024-11-16T09:58:01,813 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:37267-0x101b083d7320001, quorum=127.0.0.1:49647, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-16T09:58:01,813 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:37877-0x101b083d7320000, quorum=127.0.0.1:49647, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-16T09:58:01,813 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-16T09:58:01,813 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-16T09:58:01,813 DEBUG [PEWorker-5 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=3, ppid=2, state=RUNNABLE, hasLock=true; OpenRegionProcedure 1588230740, server=037c58a19ddb,37267,1731751077844 2024-11-16T09:58:01,818 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=3, resume processing ppid=2 2024-11-16T09:58:01,818 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=3, ppid=2, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=037c58a19ddb,37267,1731751077844 in 287 msec 2024-11-16T09:58:01,825 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=2, resume processing ppid=1 2024-11-16T09:58:01,825 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=2, ppid=1, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN in 490 msec 2024-11-16T09:58:01,826 DEBUG [PEWorker-2 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_CREATE_NAMESPACES, hasLock=true; InitMetaProcedure table=hbase:meta 2024-11-16T09:58:01,827 INFO [PEWorker-2 {}] procedure.InitMetaProcedure(114): Going to create {NAME => 'default'} and {NAME => 'hbase'} namespaces 2024-11-16T09:58:01,848 DEBUG [PEWorker-2 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-16T09:58:01,849 DEBUG [PEWorker-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=037c58a19ddb,37267,1731751077844, seqNum=-1] 2024-11-16T09:58:01,871 DEBUG [PEWorker-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-16T09:58:01,874 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:44905, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-16T09:58:01,898 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta in 1.5420 sec 2024-11-16T09:58:01,898 INFO [master/037c58a19ddb:0:becomeActiveMaster {}] master.HMaster(1123): Wait for region servers to report in: status=status unset, state=RUNNING, startTime=1731751081898, completionTime=-1 2024-11-16T09:58:01,901 INFO [master/037c58a19ddb:0:becomeActiveMaster {}] master.ServerManager(903): Finished waiting on RegionServer count=1; waited=0ms, expected min=1 server(s), max=1 server(s), master is running 2024-11-16T09:58:01,902 DEBUG [master/037c58a19ddb:0:becomeActiveMaster {}] assignment.AssignmentManager(1764): Joining cluster... 2024-11-16T09:58:01,930 INFO [master/037c58a19ddb:0:becomeActiveMaster {}] assignment.AssignmentManager(1776): Number of RegionServers=1 2024-11-16T09:58:01,930 INFO [master/037c58a19ddb:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1731751141930 2024-11-16T09:58:01,930 INFO [master/037c58a19ddb:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1731751201930 2024-11-16T09:58:01,931 INFO [master/037c58a19ddb:0:becomeActiveMaster {}] assignment.AssignmentManager(1783): Joined the cluster in 29 msec 2024-11-16T09:58:01,933 INFO [master/037c58a19ddb:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=037c58a19ddb,37877,1731751077131-ClusterStatusChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-16T09:58:01,933 INFO [master/037c58a19ddb:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=037c58a19ddb,37877,1731751077131-BalancerChore, period=300000, unit=MILLISECONDS is enabled. 2024-11-16T09:58:01,934 INFO [master/037c58a19ddb:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=037c58a19ddb,37877,1731751077131-RegionNormalizerChore, period=300000, unit=MILLISECONDS is enabled. 2024-11-16T09:58:01,935 INFO [master/037c58a19ddb:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=CatalogJanitor-037c58a19ddb:37877, period=300000, unit=MILLISECONDS is enabled. 2024-11-16T09:58:01,936 INFO [master/037c58a19ddb:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=HbckChore-, period=3600000, unit=MILLISECONDS is enabled. 2024-11-16T09:58:01,936 INFO [master/037c58a19ddb:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=FlushedSequenceIdFlusher, period=10800000, unit=MILLISECONDS is enabled. 2024-11-16T09:58:01,943 DEBUG [master/037c58a19ddb:0.Chore.1 {}] janitor.CatalogJanitor(180): 2024-11-16T09:58:01,965 INFO [master/037c58a19ddb:0:becomeActiveMaster {}] master.HMaster(1239): Master has completed initialization 3.971sec 2024-11-16T09:58:01,966 INFO [master/037c58a19ddb:0:becomeActiveMaster {}] quotas.MasterQuotaManager(97): Quota support disabled 2024-11-16T09:58:01,968 INFO [master/037c58a19ddb:0:becomeActiveMaster {}] slowlog.SlowLogMasterService(57): Slow/Large requests logging to system table hbase:slowlog is disabled. Quitting. 2024-11-16T09:58:01,969 INFO [master/037c58a19ddb:0:becomeActiveMaster {}] waleventtracker.WALEventTrackerTableCreator(75): wal event tracker requests logging to table REPLICATION.WALEVENTTRACKER is disabled. Quitting. 2024-11-16T09:58:01,969 INFO [master/037c58a19ddb:0:becomeActiveMaster {}] master.ReplicationSinkTrackerTableCreator(90): replication sink tracker requests logging to table REPLICATION.SINK_TRACKER is disabled. Quitting. 2024-11-16T09:58:01,969 INFO [master/037c58a19ddb:0:becomeActiveMaster {}] zookeeper.ZKWatcher(271): not a secure deployment, proceeding 2024-11-16T09:58:01,970 INFO [master/037c58a19ddb:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=037c58a19ddb,37877,1731751077131-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-11-16T09:58:01,971 INFO [master/037c58a19ddb:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=037c58a19ddb,37877,1731751077131-MobFileCompactionChore, period=604800, unit=SECONDS is enabled. 2024-11-16T09:58:01,980 DEBUG [master/037c58a19ddb:0:becomeActiveMaster {}] master.HMaster(1374): Balancer post startup initialization complete, took 0 seconds 2024-11-16T09:58:01,981 INFO [master/037c58a19ddb:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=RollingUpgradeChore, period=10, unit=SECONDS is enabled. 2024-11-16T09:58:01,981 INFO [master/037c58a19ddb:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=037c58a19ddb,37877,1731751077131-OldWALsDirSizeChore, period=300000, unit=MILLISECONDS is enabled. 2024-11-16T09:58:02,058 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@45544f78, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T09:58:02,060 DEBUG [Time-limited test {}] nio.NioEventLoop(110): -Dio.netty.noKeySetOptimization: false 2024-11-16T09:58:02,061 DEBUG [Time-limited test {}] nio.NioEventLoop(111): -Dio.netty.selectorAutoRebuildThreshold: 512 2024-11-16T09:58:02,064 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 037c58a19ddb,37877,-1 for getting cluster id 2024-11-16T09:58:02,067 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-16T09:58:02,075 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '1fe46ee8-cb44-44b1-8251-37103629f786' 2024-11-16T09:58:02,077 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-16T09:58:02,077 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "1fe46ee8-cb44-44b1-8251-37103629f786" 2024-11-16T09:58:02,080 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2d6faf04, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T09:58:02,080 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [037c58a19ddb,37877,-1] 2024-11-16T09:58:02,082 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-16T09:58:02,084 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T09:58:02,086 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:58536, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-16T09:58:02,088 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2de67725, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T09:58:02,089 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-16T09:58:02,096 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=037c58a19ddb,37267,1731751077844, seqNum=-1] 2024-11-16T09:58:02,096 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-16T09:58:02,099 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:39600, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-16T09:58:02,120 INFO [Time-limited test {}] hbase.HBaseTestingUtil(877): Minicluster is up; activeMaster=037c58a19ddb,37877,1731751077131 2024-11-16T09:58:02,133 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: TestAcidGuaranteesWithNoInMemCompaction#testMixedAtomicity Thread=211, OpenFileDescriptor=441, MaxFileDescriptor=1048576, SystemLoadAverage=223, ProcessCount=11, AvailableMemoryMB=5911 2024-11-16T09:58:02,158 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-11-16T09:58:02,164 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncConnectionImpl(321): The fetched master address is 037c58a19ddb,37877,1731751077131 2024-11-16T09:58:02,166 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@21bc212f 2024-11-16T09:58:02,167 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-11-16T09:58:02,170 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:58538, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-11-16T09:58:02,173 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37877 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (131072) is too small, which might cause very frequent flushing. 2024-11-16T09:58:02,177 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37877 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.2 create 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'NONE'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-16T09:58:02,189 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37877 {}] procedure2.ProcedureExecutor(1139): Stored pid=4, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=TestAcidGuarantees 2024-11-16T09:58:02,192 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_PRE_OPERATION 2024-11-16T09:58:02,194 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37877 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.2 procedure request for creating table: namespace: "default" qualifier: "TestAcidGuarantees" procId is: 4 2024-11-16T09:58:02,194 DEBUG [PEWorker-3 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T09:58:02,196 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-11-16T09:58:02,199 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37877 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-16T09:58:02,206 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37591 is added to blk_1073741835_1011 (size=959) 2024-11-16T09:58:02,314 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37877 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-16T09:58:02,523 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37877 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-16T09:58:02,611 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => d9e28756f78b0555a6733ffbf6ab98bd, NAME => 'TestAcidGuarantees,,1731751082172.d9e28756f78b0555a6733ffbf6ab98bd.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'NONE', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592 2024-11-16T09:58:02,619 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37591 is added to blk_1073741836_1012 (size=53) 2024-11-16T09:58:02,834 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37877 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-16T09:58:03,021 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1731751082172.d9e28756f78b0555a6733ffbf6ab98bd.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-16T09:58:03,022 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1722): Closing d9e28756f78b0555a6733ffbf6ab98bd, disabling compactions & flushes 2024-11-16T09:58:03,022 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1731751082172.d9e28756f78b0555a6733ffbf6ab98bd. 2024-11-16T09:58:03,022 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1731751082172.d9e28756f78b0555a6733ffbf6ab98bd. 2024-11-16T09:58:03,022 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1731751082172.d9e28756f78b0555a6733ffbf6ab98bd. after waiting 0 ms 2024-11-16T09:58:03,022 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1731751082172.d9e28756f78b0555a6733ffbf6ab98bd. 2024-11-16T09:58:03,022 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1731751082172.d9e28756f78b0555a6733ffbf6ab98bd. 2024-11-16T09:58:03,022 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1676): Region close journal for d9e28756f78b0555a6733ffbf6ab98bd: Waiting for close lock at 1731751083021Disabling compacts and flushes for region at 1731751083021Disabling writes for close at 1731751083022 (+1 ms)Writing region close event to WAL at 1731751083022Closed at 1731751083022 2024-11-16T09:58:03,024 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ADD_TO_META 2024-11-16T09:58:03,029 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"TestAcidGuarantees,,1731751082172.d9e28756f78b0555a6733ffbf6ab98bd.","families":{"info":[{"qualifier":"regioninfo","vlen":52,"tag":[],"timestamp":"1731751083025"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1731751083025"}]},"ts":"1731751083025"} 2024-11-16T09:58:03,037 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-11-16T09:58:03,039 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-11-16T09:58:03,042 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731751083040"}]},"ts":"1731751083040"} 2024-11-16T09:58:03,046 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLING in hbase:meta 2024-11-16T09:58:03,048 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=d9e28756f78b0555a6733ffbf6ab98bd, ASSIGN}] 2024-11-16T09:58:03,050 INFO [PEWorker-4 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=d9e28756f78b0555a6733ffbf6ab98bd, ASSIGN 2024-11-16T09:58:03,052 INFO [PEWorker-4 {}] assignment.TransitRegionStateProcedure(269): Starting pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=d9e28756f78b0555a6733ffbf6ab98bd, ASSIGN; state=OFFLINE, location=037c58a19ddb,37267,1731751077844; forceNewPlan=false, retain=false 2024-11-16T09:58:03,204 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=5 updating hbase:meta row=d9e28756f78b0555a6733ffbf6ab98bd, regionState=OPENING, regionLocation=037c58a19ddb,37267,1731751077844 2024-11-16T09:58:03,209 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=d9e28756f78b0555a6733ffbf6ab98bd, ASSIGN because future has completed 2024-11-16T09:58:03,210 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=6, ppid=5, state=RUNNABLE, hasLock=false; OpenRegionProcedure d9e28756f78b0555a6733ffbf6ab98bd, server=037c58a19ddb,37267,1731751077844}] 2024-11-16T09:58:03,344 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37877 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-16T09:58:03,371 INFO [RS_OPEN_REGION-regionserver/037c58a19ddb:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] handler.AssignRegionHandler(132): Open TestAcidGuarantees,,1731751082172.d9e28756f78b0555a6733ffbf6ab98bd. 2024-11-16T09:58:03,371 DEBUG [RS_OPEN_REGION-regionserver/037c58a19ddb:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7752): Opening region: {ENCODED => d9e28756f78b0555a6733ffbf6ab98bd, NAME => 'TestAcidGuarantees,,1731751082172.d9e28756f78b0555a6733ffbf6ab98bd.', STARTKEY => '', ENDKEY => ''} 2024-11-16T09:58:03,372 DEBUG [RS_OPEN_REGION-regionserver/037c58a19ddb:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestAcidGuarantees d9e28756f78b0555a6733ffbf6ab98bd 2024-11-16T09:58:03,372 DEBUG [RS_OPEN_REGION-regionserver/037c58a19ddb:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1731751082172.d9e28756f78b0555a6733ffbf6ab98bd.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-16T09:58:03,372 DEBUG [RS_OPEN_REGION-regionserver/037c58a19ddb:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7794): checking encryption for d9e28756f78b0555a6733ffbf6ab98bd 2024-11-16T09:58:03,372 DEBUG [RS_OPEN_REGION-regionserver/037c58a19ddb:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7797): checking classloading for d9e28756f78b0555a6733ffbf6ab98bd 2024-11-16T09:58:03,374 INFO [StoreOpener-d9e28756f78b0555a6733ffbf6ab98bd-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family A of region d9e28756f78b0555a6733ffbf6ab98bd 2024-11-16T09:58:03,377 INFO [StoreOpener-d9e28756f78b0555a6733ffbf6ab98bd-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region d9e28756f78b0555a6733ffbf6ab98bd columnFamilyName A 2024-11-16T09:58:03,377 DEBUG [StoreOpener-d9e28756f78b0555a6733ffbf6ab98bd-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T09:58:03,378 INFO [StoreOpener-d9e28756f78b0555a6733ffbf6ab98bd-1 {}] regionserver.HStore(327): Store=d9e28756f78b0555a6733ffbf6ab98bd/A, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-16T09:58:03,378 INFO [StoreOpener-d9e28756f78b0555a6733ffbf6ab98bd-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family B of region d9e28756f78b0555a6733ffbf6ab98bd 2024-11-16T09:58:03,381 INFO [StoreOpener-d9e28756f78b0555a6733ffbf6ab98bd-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region d9e28756f78b0555a6733ffbf6ab98bd columnFamilyName B 2024-11-16T09:58:03,381 DEBUG [StoreOpener-d9e28756f78b0555a6733ffbf6ab98bd-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T09:58:03,382 INFO [StoreOpener-d9e28756f78b0555a6733ffbf6ab98bd-1 {}] regionserver.HStore(327): Store=d9e28756f78b0555a6733ffbf6ab98bd/B, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-16T09:58:03,382 INFO [StoreOpener-d9e28756f78b0555a6733ffbf6ab98bd-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family C of region d9e28756f78b0555a6733ffbf6ab98bd 2024-11-16T09:58:03,384 INFO [StoreOpener-d9e28756f78b0555a6733ffbf6ab98bd-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region d9e28756f78b0555a6733ffbf6ab98bd columnFamilyName C 2024-11-16T09:58:03,384 DEBUG [StoreOpener-d9e28756f78b0555a6733ffbf6ab98bd-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T09:58:03,385 INFO [StoreOpener-d9e28756f78b0555a6733ffbf6ab98bd-1 {}] regionserver.HStore(327): Store=d9e28756f78b0555a6733ffbf6ab98bd/C, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-16T09:58:03,385 DEBUG [RS_OPEN_REGION-regionserver/037c58a19ddb:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1038): replaying wal for d9e28756f78b0555a6733ffbf6ab98bd 2024-11-16T09:58:03,387 DEBUG [RS_OPEN_REGION-regionserver/037c58a19ddb:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/d9e28756f78b0555a6733ffbf6ab98bd 2024-11-16T09:58:03,387 DEBUG [RS_OPEN_REGION-regionserver/037c58a19ddb:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/d9e28756f78b0555a6733ffbf6ab98bd 2024-11-16T09:58:03,389 DEBUG [RS_OPEN_REGION-regionserver/037c58a19ddb:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1048): stopping wal replay for d9e28756f78b0555a6733ffbf6ab98bd 2024-11-16T09:58:03,389 DEBUG [RS_OPEN_REGION-regionserver/037c58a19ddb:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1060): Cleaning up temporary data for d9e28756f78b0555a6733ffbf6ab98bd 2024-11-16T09:58:03,390 DEBUG [RS_OPEN_REGION-regionserver/037c58a19ddb:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table TestAcidGuarantees descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-11-16T09:58:03,392 DEBUG [RS_OPEN_REGION-regionserver/037c58a19ddb:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1093): writing seq id for d9e28756f78b0555a6733ffbf6ab98bd 2024-11-16T09:58:03,396 DEBUG [RS_OPEN_REGION-regionserver/037c58a19ddb:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/d9e28756f78b0555a6733ffbf6ab98bd/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-16T09:58:03,397 INFO [RS_OPEN_REGION-regionserver/037c58a19ddb:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1114): Opened d9e28756f78b0555a6733ffbf6ab98bd; next sequenceid=2; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=67080888, jitterRate=-4.1687488555908203E-4}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-11-16T09:58:03,397 DEBUG [RS_OPEN_REGION-regionserver/037c58a19ddb:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1122): Running coprocessor post-open hooks for d9e28756f78b0555a6733ffbf6ab98bd 2024-11-16T09:58:03,398 DEBUG [RS_OPEN_REGION-regionserver/037c58a19ddb:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1006): Region open journal for d9e28756f78b0555a6733ffbf6ab98bd: Running coprocessor pre-open hook at 1731751083372Writing region info on filesystem at 1731751083372Initializing all the Stores at 1731751083374 (+2 ms)Instantiating store for column family {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731751083374Instantiating store for column family {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731751083374Instantiating store for column family {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731751083374Cleaning up temporary data from old regions at 1731751083389 (+15 ms)Running coprocessor post-open hooks at 1731751083397 (+8 ms)Region opened successfully at 1731751083398 (+1 ms) 2024-11-16T09:58:03,400 INFO [RS_OPEN_REGION-regionserver/037c58a19ddb:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegionServer(2236): Post open deploy tasks for TestAcidGuarantees,,1731751082172.d9e28756f78b0555a6733ffbf6ab98bd., pid=6, masterSystemTime=1731751083364 2024-11-16T09:58:03,404 DEBUG [RS_OPEN_REGION-regionserver/037c58a19ddb:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegionServer(2266): Finished post open deploy task for TestAcidGuarantees,,1731751082172.d9e28756f78b0555a6733ffbf6ab98bd. 2024-11-16T09:58:03,404 INFO [RS_OPEN_REGION-regionserver/037c58a19ddb:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] handler.AssignRegionHandler(153): Opened TestAcidGuarantees,,1731751082172.d9e28756f78b0555a6733ffbf6ab98bd. 2024-11-16T09:58:03,405 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=5 updating hbase:meta row=d9e28756f78b0555a6733ffbf6ab98bd, regionState=OPEN, openSeqNum=2, regionLocation=037c58a19ddb,37267,1731751077844 2024-11-16T09:58:03,410 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=6, ppid=5, state=RUNNABLE, hasLock=false; OpenRegionProcedure d9e28756f78b0555a6733ffbf6ab98bd, server=037c58a19ddb,37267,1731751077844 because future has completed 2024-11-16T09:58:03,417 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=6, resume processing ppid=5 2024-11-16T09:58:03,417 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=6, ppid=5, state=SUCCESS, hasLock=false; OpenRegionProcedure d9e28756f78b0555a6733ffbf6ab98bd, server=037c58a19ddb,37267,1731751077844 in 202 msec 2024-11-16T09:58:03,421 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=5, resume processing ppid=4 2024-11-16T09:58:03,422 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=5, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=d9e28756f78b0555a6733ffbf6ab98bd, ASSIGN in 369 msec 2024-11-16T09:58:03,423 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-11-16T09:58:03,423 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731751083423"}]},"ts":"1731751083423"} 2024-11-16T09:58:03,427 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLED in hbase:meta 2024-11-16T09:58:03,428 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_POST_OPERATION 2024-11-16T09:58:03,431 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=TestAcidGuarantees in 1.2440 sec 2024-11-16T09:58:04,354 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37877 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-16T09:58:04,355 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: default:TestAcidGuarantees completed 2024-11-16T09:58:04,360 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@19b2c994, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T09:58:04,360 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 037c58a19ddb,37877,-1 for getting cluster id 2024-11-16T09:58:04,361 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-16T09:58:04,362 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '1fe46ee8-cb44-44b1-8251-37103629f786' 2024-11-16T09:58:04,363 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-16T09:58:04,363 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "1fe46ee8-cb44-44b1-8251-37103629f786" 2024-11-16T09:58:04,363 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2c0df3b8, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T09:58:04,363 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [037c58a19ddb,37877,-1] 2024-11-16T09:58:04,364 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-16T09:58:04,364 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T09:58:04,366 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:58558, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-16T09:58:04,367 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@75911b03, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T09:58:04,367 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-16T09:58:04,369 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=037c58a19ddb,37267,1731751077844, seqNum=-1] 2024-11-16T09:58:04,369 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-16T09:58:04,371 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:39612, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-16T09:58:04,374 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-11-16T09:58:04,375 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncConnectionImpl(321): The fetched master address is 037c58a19ddb,37877,1731751077131 2024-11-16T09:58:04,375 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@6187fcd5 2024-11-16T09:58:04,377 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-11-16T09:58:04,379 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:58562, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-11-16T09:58:04,383 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6a52ffa9, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T09:58:04,383 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 037c58a19ddb,37877,-1 for getting cluster id 2024-11-16T09:58:04,383 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-16T09:58:04,384 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '1fe46ee8-cb44-44b1-8251-37103629f786' 2024-11-16T09:58:04,385 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-16T09:58:04,385 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "1fe46ee8-cb44-44b1-8251-37103629f786" 2024-11-16T09:58:04,385 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@39f7011d, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T09:58:04,385 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [037c58a19ddb,37877,-1] 2024-11-16T09:58:04,385 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-16T09:58:04,386 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T09:58:04,387 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:58574, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-16T09:58:04,388 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4834e559, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T09:58:04,389 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@51f14cda, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T09:58:04,389 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 037c58a19ddb,37877,-1 for getting cluster id 2024-11-16T09:58:04,389 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-16T09:58:04,390 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '1fe46ee8-cb44-44b1-8251-37103629f786' 2024-11-16T09:58:04,391 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-16T09:58:04,391 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "1fe46ee8-cb44-44b1-8251-37103629f786" 2024-11-16T09:58:04,391 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@44447930, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T09:58:04,391 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [037c58a19ddb,37877,-1] 2024-11-16T09:58:04,391 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-16T09:58:04,392 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T09:58:04,393 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:58598, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-16T09:58:04,394 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2ac8d8d, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T09:58:04,395 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4201ea29, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T09:58:04,395 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 037c58a19ddb,37877,-1 for getting cluster id 2024-11-16T09:58:04,396 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-16T09:58:04,397 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '1fe46ee8-cb44-44b1-8251-37103629f786' 2024-11-16T09:58:04,397 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-16T09:58:04,397 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "1fe46ee8-cb44-44b1-8251-37103629f786" 2024-11-16T09:58:04,398 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@797de42a, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T09:58:04,398 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [037c58a19ddb,37877,-1] 2024-11-16T09:58:04,398 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-16T09:58:04,400 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T09:58:04,402 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:58616, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-16T09:58:04,403 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6da9de27, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T09:58:04,403 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@45cc41cd, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T09:58:04,403 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 037c58a19ddb,37877,-1 for getting cluster id 2024-11-16T09:58:04,404 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-16T09:58:04,405 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '1fe46ee8-cb44-44b1-8251-37103629f786' 2024-11-16T09:58:04,405 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-16T09:58:04,405 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "1fe46ee8-cb44-44b1-8251-37103629f786" 2024-11-16T09:58:04,406 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7a1e59f6, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T09:58:04,406 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [037c58a19ddb,37877,-1] 2024-11-16T09:58:04,406 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-16T09:58:04,406 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T09:58:04,408 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:58638, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-16T09:58:04,409 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@31087b58, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T09:58:04,410 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2900b22, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T09:58:04,410 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 037c58a19ddb,37877,-1 for getting cluster id 2024-11-16T09:58:04,410 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-16T09:58:04,411 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '1fe46ee8-cb44-44b1-8251-37103629f786' 2024-11-16T09:58:04,411 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-16T09:58:04,411 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "1fe46ee8-cb44-44b1-8251-37103629f786" 2024-11-16T09:58:04,412 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4712ff10, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T09:58:04,412 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [037c58a19ddb,37877,-1] 2024-11-16T09:58:04,412 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-16T09:58:04,412 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T09:58:04,413 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:58656, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-16T09:58:04,414 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@33fb2726, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T09:58:04,415 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5d39686f, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T09:58:04,415 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 037c58a19ddb,37877,-1 for getting cluster id 2024-11-16T09:58:04,415 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-16T09:58:04,416 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '1fe46ee8-cb44-44b1-8251-37103629f786' 2024-11-16T09:58:04,417 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-16T09:58:04,417 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "1fe46ee8-cb44-44b1-8251-37103629f786" 2024-11-16T09:58:04,418 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@14983026, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T09:58:04,418 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [037c58a19ddb,37877,-1] 2024-11-16T09:58:04,418 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-16T09:58:04,418 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T09:58:04,419 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:58666, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-16T09:58:04,420 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4c7a39e, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T09:58:04,421 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6963d59b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T09:58:04,421 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 037c58a19ddb,37877,-1 for getting cluster id 2024-11-16T09:58:04,421 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-16T09:58:04,422 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '1fe46ee8-cb44-44b1-8251-37103629f786' 2024-11-16T09:58:04,422 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-16T09:58:04,422 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "1fe46ee8-cb44-44b1-8251-37103629f786" 2024-11-16T09:58:04,423 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@16f2ee53, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T09:58:04,423 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [037c58a19ddb,37877,-1] 2024-11-16T09:58:04,423 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-16T09:58:04,423 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T09:58:04,424 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:58678, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-16T09:58:04,425 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@580da22a, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T09:58:04,426 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2b9a31d1, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T09:58:04,426 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 037c58a19ddb,37877,-1 for getting cluster id 2024-11-16T09:58:04,427 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-16T09:58:04,427 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '1fe46ee8-cb44-44b1-8251-37103629f786' 2024-11-16T09:58:04,428 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-16T09:58:04,428 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "1fe46ee8-cb44-44b1-8251-37103629f786" 2024-11-16T09:58:04,428 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@707607ee, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T09:58:04,428 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [037c58a19ddb,37877,-1] 2024-11-16T09:58:04,428 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-16T09:58:04,428 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T09:58:04,429 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:58706, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-16T09:58:04,430 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1ac02c9a, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T09:58:04,431 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@f0b495b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T09:58:04,431 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 037c58a19ddb,37877,-1 for getting cluster id 2024-11-16T09:58:04,431 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-16T09:58:04,432 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '1fe46ee8-cb44-44b1-8251-37103629f786' 2024-11-16T09:58:04,432 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-16T09:58:04,432 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "1fe46ee8-cb44-44b1-8251-37103629f786" 2024-11-16T09:58:04,432 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2c7642ac, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T09:58:04,433 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [037c58a19ddb,37877,-1] 2024-11-16T09:58:04,433 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-16T09:58:04,433 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T09:58:04,434 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:58714, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-16T09:58:04,435 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1d66fe7a, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T09:58:04,439 DEBUG [Thread-147 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-16T09:58:04,439 DEBUG [Thread-143 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-16T09:58:04,439 DEBUG [Thread-160 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-16T09:58:04,439 DEBUG [Thread-151 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-16T09:58:04,440 DEBUG [Thread-149 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-16T09:58:04,440 DEBUG [Thread-158 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-16T09:58:04,442 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=037c58a19ddb,37267,1731751077844, seqNum=-1] 2024-11-16T09:58:04,442 DEBUG [Thread-151 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=037c58a19ddb,37267,1731751077844, seqNum=-1] 2024-11-16T09:58:04,442 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=037c58a19ddb,37267,1731751077844, seqNum=-1] 2024-11-16T09:58:04,442 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-16T09:58:04,442 DEBUG [Thread-151 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-16T09:58:04,442 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-16T09:58:04,443 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=037c58a19ddb,37267,1731751077844, seqNum=-1] 2024-11-16T09:58:04,444 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-16T09:58:04,444 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=037c58a19ddb,37267,1731751077844, seqNum=-1] 2024-11-16T09:58:04,444 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-16T09:58:04,444 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:39620, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-16T09:58:04,444 DEBUG [Thread-156 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-16T09:58:04,445 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:39630, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-16T09:58:04,445 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:39616, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-16T09:58:04,446 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=037c58a19ddb,37267,1731751077844, seqNum=-1] 2024-11-16T09:58:04,446 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:39638, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-16T09:58:04,446 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-16T09:58:04,447 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:39654, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-16T09:58:04,448 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:39670, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-16T09:58:04,449 DEBUG [Thread-145 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-16T09:58:04,449 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37877 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-16T09:58:04,452 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1731751082172.d9e28756f78b0555a6733ffbf6ab98bd., hostname=037c58a19ddb,37267,1731751077844, seqNum=2] 2024-11-16T09:58:04,453 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1731751082172.d9e28756f78b0555a6733ffbf6ab98bd., hostname=037c58a19ddb,37267,1731751077844, seqNum=2] 2024-11-16T09:58:04,454 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1731751082172.d9e28756f78b0555a6733ffbf6ab98bd., hostname=037c58a19ddb,37267,1731751077844, seqNum=2] 2024-11-16T09:58:04,455 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1731751082172.d9e28756f78b0555a6733ffbf6ab98bd., hostname=037c58a19ddb,37267,1731751077844, seqNum=2] 2024-11-16T09:58:04,456 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1731751082172.d9e28756f78b0555a6733ffbf6ab98bd., hostname=037c58a19ddb,37267,1731751077844, seqNum=2] 2024-11-16T09:58:04,457 DEBUG [Thread-154 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-16T09:58:04,458 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37877 {}] procedure2.ProcedureExecutor(1139): Stored pid=7, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=7, table=TestAcidGuarantees 2024-11-16T09:58:04,458 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=037c58a19ddb,37267,1731751077844, seqNum=-1] 2024-11-16T09:58:04,459 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=037c58a19ddb,37267,1731751077844, seqNum=-1] 2024-11-16T09:58:04,459 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-16T09:58:04,459 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-16T09:58:04,459 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1731751082172.d9e28756f78b0555a6733ffbf6ab98bd., hostname=037c58a19ddb,37267,1731751077844, seqNum=2] 2024-11-16T09:58:04,461 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=037c58a19ddb,37267,1731751077844, seqNum=-1] 2024-11-16T09:58:04,461 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-16T09:58:04,463 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37877 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=7 2024-11-16T09:58:04,465 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:39704, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-16T09:58:04,465 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=7, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=7, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-16T09:58:04,466 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:39684, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-16T09:58:04,468 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=7, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=7, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-16T09:58:04,468 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1731751082172.d9e28756f78b0555a6733ffbf6ab98bd., hostname=037c58a19ddb,37267,1731751077844, seqNum=2] 2024-11-16T09:58:04,469 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:39696, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-16T09:58:04,469 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=8, ppid=7, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-16T09:58:04,470 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1731751082172.d9e28756f78b0555a6733ffbf6ab98bd., hostname=037c58a19ddb,37267,1731751077844, seqNum=2] 2024-11-16T09:58:04,473 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1731751082172.d9e28756f78b0555a6733ffbf6ab98bd., hostname=037c58a19ddb,37267,1731751077844, seqNum=2] 2024-11-16T09:58:04,523 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-16T09:58:04,524 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] regionserver.HRegion(8855): Flush requested on d9e28756f78b0555a6733ffbf6ab98bd 2024-11-16T09:58:04,528 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing d9e28756f78b0555a6733ffbf6ab98bd 3/3 column families, dataSize=53.67 KB heapSize=141.38 KB 2024-11-16T09:58:04,573 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37877 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=7 2024-11-16T09:58:04,636 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=37267 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=8 2024-11-16T09:58:04,637 DEBUG [RS_FLUSH_OPERATIONS-regionserver/037c58a19ddb:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731751082172.d9e28756f78b0555a6733ffbf6ab98bd. 2024-11-16T09:58:04,637 DEBUG [RS_FLUSH_OPERATIONS-regionserver/037c58a19ddb:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731751082172.d9e28756f78b0555a6733ffbf6ab98bd. as already flushing 2024-11-16T09:58:04,638 DEBUG [RS_FLUSH_OPERATIONS-regionserver/037c58a19ddb:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731751082172.d9e28756f78b0555a6733ffbf6ab98bd. 2024-11-16T09:58:04,639 ERROR [RS_FLUSH_OPERATIONS-regionserver/037c58a19ddb:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] handler.RSProcedureHandler(58): pid=8 java.io.IOException: Unable to complete flush {ENCODED => d9e28756f78b0555a6733ffbf6ab98bd, NAME => 'TestAcidGuarantees,,1731751082172.d9e28756f78b0555a6733ffbf6ab98bd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T09:58:04,643 DEBUG [RS_FLUSH_OPERATIONS-regionserver/037c58a19ddb:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=8 java.io.IOException: Unable to complete flush {ENCODED => d9e28756f78b0555a6733ffbf6ab98bd, NAME => 'TestAcidGuarantees,,1731751082172.d9e28756f78b0555a6733ffbf6ab98bd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T09:58:04,646 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/d9e28756f78b0555a6733ffbf6ab98bd/.tmp/A/08ff50e864824ad0a3fd6cbd2d93846b is 50, key is test_row_0/A:col10/1731751084509/Put/seqid=0 2024-11-16T09:58:04,668 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37877 {}] master.HMaster(4177): Remote procedure failed, pid=8 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d9e28756f78b0555a6733ffbf6ab98bd, NAME => 'TestAcidGuarantees,,1731751082172.d9e28756f78b0555a6733ffbf6ab98bd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d9e28756f78b0555a6733ffbf6ab98bd, NAME => 'TestAcidGuarantees,,1731751082172.d9e28756f78b0555a6733ffbf6ab98bd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T09:58:04,684 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37591 is added to blk_1073741837_1013 (size=12001) 2024-11-16T09:58:04,687 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=13 (bloomFilter=true), to=hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/d9e28756f78b0555a6733ffbf6ab98bd/.tmp/A/08ff50e864824ad0a3fd6cbd2d93846b 2024-11-16T09:58:04,695 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9e28756f78b0555a6733ffbf6ab98bd, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T09:58:04,701 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39654 deadline: 1731751144684, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9e28756f78b0555a6733ffbf6ab98bd, server=037c58a19ddb,37267,1731751077844 2024-11-16T09:58:04,702 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9e28756f78b0555a6733ffbf6ab98bd, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T09:58:04,703 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39620 deadline: 1731751144692, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9e28756f78b0555a6733ffbf6ab98bd, server=037c58a19ddb,37267,1731751077844 2024-11-16T09:58:04,703 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731751082172.d9e28756f78b0555a6733ffbf6ab98bd., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 , the old value is region=TestAcidGuarantees,,1731751082172.d9e28756f78b0555a6733ffbf6ab98bd., hostname=037c58a19ddb,37267,1731751077844, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9e28756f78b0555a6733ffbf6ab98bd, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:58:04,704 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731751082172.d9e28756f78b0555a6733ffbf6ab98bd., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 , the old value is region=TestAcidGuarantees,,1731751082172.d9e28756f78b0555a6733ffbf6ab98bd., hostname=037c58a19ddb,37267,1731751077844, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9e28756f78b0555a6733ffbf6ab98bd, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:58:04,704 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731751082172.d9e28756f78b0555a6733ffbf6ab98bd., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9e28756f78b0555a6733ffbf6ab98bd, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:58:04,704 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731751082172.d9e28756f78b0555a6733ffbf6ab98bd., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9e28756f78b0555a6733ffbf6ab98bd, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:58:04,704 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731751082172.d9e28756f78b0555a6733ffbf6ab98bd., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 because the exception is null or not the one we care about 2024-11-16T09:58:04,704 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731751082172.d9e28756f78b0555a6733ffbf6ab98bd., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 because the exception is null or not the one we care about 2024-11-16T09:58:04,718 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9e28756f78b0555a6733ffbf6ab98bd, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T09:58:04,718 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39630 deadline: 1731751144696, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9e28756f78b0555a6733ffbf6ab98bd, server=037c58a19ddb,37267,1731751077844 2024-11-16T09:58:04,719 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731751082172.d9e28756f78b0555a6733ffbf6ab98bd., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 , the old value is region=TestAcidGuarantees,,1731751082172.d9e28756f78b0555a6733ffbf6ab98bd., hostname=037c58a19ddb,37267,1731751077844, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9e28756f78b0555a6733ffbf6ab98bd, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:58:04,719 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731751082172.d9e28756f78b0555a6733ffbf6ab98bd., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9e28756f78b0555a6733ffbf6ab98bd, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:58:04,719 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9e28756f78b0555a6733ffbf6ab98bd, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T09:58:04,719 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731751082172.d9e28756f78b0555a6733ffbf6ab98bd., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 because the exception is null or not the one we care about 2024-11-16T09:58:04,720 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39638 deadline: 1731751144703, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9e28756f78b0555a6733ffbf6ab98bd, server=037c58a19ddb,37267,1731751077844 2024-11-16T09:58:04,721 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731751082172.d9e28756f78b0555a6733ffbf6ab98bd., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 , the old value is region=TestAcidGuarantees,,1731751082172.d9e28756f78b0555a6733ffbf6ab98bd., hostname=037c58a19ddb,37267,1731751077844, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9e28756f78b0555a6733ffbf6ab98bd, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:58:04,721 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731751082172.d9e28756f78b0555a6733ffbf6ab98bd., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9e28756f78b0555a6733ffbf6ab98bd, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:58:04,721 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731751082172.d9e28756f78b0555a6733ffbf6ab98bd., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 because the exception is null or not the one we care about 2024-11-16T09:58:04,721 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9e28756f78b0555a6733ffbf6ab98bd, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T09:58:04,722 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39696 deadline: 1731751144705, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9e28756f78b0555a6733ffbf6ab98bd, server=037c58a19ddb,37267,1731751077844 2024-11-16T09:58:04,722 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731751082172.d9e28756f78b0555a6733ffbf6ab98bd., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 , the old value is region=TestAcidGuarantees,,1731751082172.d9e28756f78b0555a6733ffbf6ab98bd., hostname=037c58a19ddb,37267,1731751077844, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9e28756f78b0555a6733ffbf6ab98bd, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:58:04,722 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731751082172.d9e28756f78b0555a6733ffbf6ab98bd., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9e28756f78b0555a6733ffbf6ab98bd, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:58:04,723 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731751082172.d9e28756f78b0555a6733ffbf6ab98bd., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 because the exception is null or not the one we care about 2024-11-16T09:58:04,783 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37877 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=7 2024-11-16T09:58:04,809 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/d9e28756f78b0555a6733ffbf6ab98bd/.tmp/B/90cd50f304d648e2bb3f087e808ef5ce is 50, key is test_row_0/B:col10/1731751084509/Put/seqid=0 2024-11-16T09:58:04,815 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9e28756f78b0555a6733ffbf6ab98bd, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T09:58:04,816 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39654 deadline: 1731751144814, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9e28756f78b0555a6733ffbf6ab98bd, server=037c58a19ddb,37267,1731751077844 2024-11-16T09:58:04,817 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731751082172.d9e28756f78b0555a6733ffbf6ab98bd., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 , the old value is region=TestAcidGuarantees,,1731751082172.d9e28756f78b0555a6733ffbf6ab98bd., hostname=037c58a19ddb,37267,1731751077844, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9e28756f78b0555a6733ffbf6ab98bd, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:58:04,817 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731751082172.d9e28756f78b0555a6733ffbf6ab98bd., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9e28756f78b0555a6733ffbf6ab98bd, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:58:04,817 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731751082172.d9e28756f78b0555a6733ffbf6ab98bd., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 because the exception is null or not the one we care about 2024-11-16T09:58:04,819 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9e28756f78b0555a6733ffbf6ab98bd, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T09:58:04,820 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39620 deadline: 1731751144814, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9e28756f78b0555a6733ffbf6ab98bd, server=037c58a19ddb,37267,1731751077844 2024-11-16T09:58:04,823 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=37267 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=8 2024-11-16T09:58:04,826 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731751082172.d9e28756f78b0555a6733ffbf6ab98bd., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 , the old value is region=TestAcidGuarantees,,1731751082172.d9e28756f78b0555a6733ffbf6ab98bd., hostname=037c58a19ddb,37267,1731751077844, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9e28756f78b0555a6733ffbf6ab98bd, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:58:04,826 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731751082172.d9e28756f78b0555a6733ffbf6ab98bd., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9e28756f78b0555a6733ffbf6ab98bd, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:58:04,826 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731751082172.d9e28756f78b0555a6733ffbf6ab98bd., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 because the exception is null or not the one we care about 2024-11-16T09:58:04,830 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9e28756f78b0555a6733ffbf6ab98bd, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T09:58:04,830 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39630 deadline: 1731751144824, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9e28756f78b0555a6733ffbf6ab98bd, server=037c58a19ddb,37267,1731751077844 2024-11-16T09:58:04,831 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731751082172.d9e28756f78b0555a6733ffbf6ab98bd., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 , the old value is region=TestAcidGuarantees,,1731751082172.d9e28756f78b0555a6733ffbf6ab98bd., hostname=037c58a19ddb,37267,1731751077844, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9e28756f78b0555a6733ffbf6ab98bd, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:58:04,831 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731751082172.d9e28756f78b0555a6733ffbf6ab98bd., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9e28756f78b0555a6733ffbf6ab98bd, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:58:04,831 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731751082172.d9e28756f78b0555a6733ffbf6ab98bd., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 because the exception is null or not the one we care about 2024-11-16T09:58:04,831 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9e28756f78b0555a6733ffbf6ab98bd, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T09:58:04,832 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39638 deadline: 1731751144824, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9e28756f78b0555a6733ffbf6ab98bd, server=037c58a19ddb,37267,1731751077844 2024-11-16T09:58:04,833 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731751082172.d9e28756f78b0555a6733ffbf6ab98bd., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 , the old value is region=TestAcidGuarantees,,1731751082172.d9e28756f78b0555a6733ffbf6ab98bd., hostname=037c58a19ddb,37267,1731751077844, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9e28756f78b0555a6733ffbf6ab98bd, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:58:04,833 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731751082172.d9e28756f78b0555a6733ffbf6ab98bd., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9e28756f78b0555a6733ffbf6ab98bd, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:58:04,833 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731751082172.d9e28756f78b0555a6733ffbf6ab98bd., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 because the exception is null or not the one we care about 2024-11-16T09:58:04,834 DEBUG [RS_FLUSH_OPERATIONS-regionserver/037c58a19ddb:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731751082172.d9e28756f78b0555a6733ffbf6ab98bd. 2024-11-16T09:58:04,834 DEBUG [RS_FLUSH_OPERATIONS-regionserver/037c58a19ddb:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731751082172.d9e28756f78b0555a6733ffbf6ab98bd. as already flushing 2024-11-16T09:58:04,834 DEBUG [RS_FLUSH_OPERATIONS-regionserver/037c58a19ddb:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731751082172.d9e28756f78b0555a6733ffbf6ab98bd. 2024-11-16T09:58:04,834 ERROR [RS_FLUSH_OPERATIONS-regionserver/037c58a19ddb:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] handler.RSProcedureHandler(58): pid=8 java.io.IOException: Unable to complete flush {ENCODED => d9e28756f78b0555a6733ffbf6ab98bd, NAME => 'TestAcidGuarantees,,1731751082172.d9e28756f78b0555a6733ffbf6ab98bd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T09:58:04,835 DEBUG [RS_FLUSH_OPERATIONS-regionserver/037c58a19ddb:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=8 java.io.IOException: Unable to complete flush {ENCODED => d9e28756f78b0555a6733ffbf6ab98bd, NAME => 'TestAcidGuarantees,,1731751082172.d9e28756f78b0555a6733ffbf6ab98bd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T09:58:04,836 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37591 is added to blk_1073741838_1014 (size=12001) 2024-11-16T09:58:04,836 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37877 {}] master.HMaster(4177): Remote procedure failed, pid=8 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d9e28756f78b0555a6733ffbf6ab98bd, NAME => 'TestAcidGuarantees,,1731751082172.d9e28756f78b0555a6733ffbf6ab98bd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d9e28756f78b0555a6733ffbf6ab98bd, NAME => 'TestAcidGuarantees,,1731751082172.d9e28756f78b0555a6733ffbf6ab98bd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T09:58:04,838 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9e28756f78b0555a6733ffbf6ab98bd, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T09:58:04,839 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39696 deadline: 1731751144834, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9e28756f78b0555a6733ffbf6ab98bd, server=037c58a19ddb,37267,1731751077844 2024-11-16T09:58:04,839 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731751082172.d9e28756f78b0555a6733ffbf6ab98bd., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 , the old value is region=TestAcidGuarantees,,1731751082172.d9e28756f78b0555a6733ffbf6ab98bd., hostname=037c58a19ddb,37267,1731751077844, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9e28756f78b0555a6733ffbf6ab98bd, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:58:04,840 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731751082172.d9e28756f78b0555a6733ffbf6ab98bd., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9e28756f78b0555a6733ffbf6ab98bd, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:58:04,840 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731751082172.d9e28756f78b0555a6733ffbf6ab98bd., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 because the exception is null or not the one we care about 2024-11-16T09:58:04,845 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=13 (bloomFilter=true), to=hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/d9e28756f78b0555a6733ffbf6ab98bd/.tmp/B/90cd50f304d648e2bb3f087e808ef5ce 2024-11-16T09:58:04,887 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/d9e28756f78b0555a6733ffbf6ab98bd/.tmp/C/0eabc0bd610f4b8fb928684c5386127e is 50, key is test_row_0/C:col10/1731751084509/Put/seqid=0 2024-11-16T09:58:04,910 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37591 is added to blk_1073741839_1015 (size=12001) 2024-11-16T09:58:04,912 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=13 (bloomFilter=true), to=hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/d9e28756f78b0555a6733ffbf6ab98bd/.tmp/C/0eabc0bd610f4b8fb928684c5386127e 2024-11-16T09:58:04,930 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/d9e28756f78b0555a6733ffbf6ab98bd/.tmp/A/08ff50e864824ad0a3fd6cbd2d93846b as hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/d9e28756f78b0555a6733ffbf6ab98bd/A/08ff50e864824ad0a3fd6cbd2d93846b 2024-11-16T09:58:04,945 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/d9e28756f78b0555a6733ffbf6ab98bd/A/08ff50e864824ad0a3fd6cbd2d93846b, entries=150, sequenceid=13, filesize=11.7 K 2024-11-16T09:58:04,953 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/d9e28756f78b0555a6733ffbf6ab98bd/.tmp/B/90cd50f304d648e2bb3f087e808ef5ce as hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/d9e28756f78b0555a6733ffbf6ab98bd/B/90cd50f304d648e2bb3f087e808ef5ce 2024-11-16T09:58:04,967 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/d9e28756f78b0555a6733ffbf6ab98bd/B/90cd50f304d648e2bb3f087e808ef5ce, entries=150, sequenceid=13, filesize=11.7 K 2024-11-16T09:58:04,970 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/d9e28756f78b0555a6733ffbf6ab98bd/.tmp/C/0eabc0bd610f4b8fb928684c5386127e as hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/d9e28756f78b0555a6733ffbf6ab98bd/C/0eabc0bd610f4b8fb928684c5386127e 2024-11-16T09:58:04,990 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/d9e28756f78b0555a6733ffbf6ab98bd/C/0eabc0bd610f4b8fb928684c5386127e, entries=150, sequenceid=13, filesize=11.7 K 2024-11-16T09:58:04,992 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=37267 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=8 2024-11-16T09:58:04,998 DEBUG [RS_FLUSH_OPERATIONS-regionserver/037c58a19ddb:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731751082172.d9e28756f78b0555a6733ffbf6ab98bd. 2024-11-16T09:58:04,999 DEBUG [RS_FLUSH_OPERATIONS-regionserver/037c58a19ddb:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731751082172.d9e28756f78b0555a6733ffbf6ab98bd. as already flushing 2024-11-16T09:58:04,999 DEBUG [RS_FLUSH_OPERATIONS-regionserver/037c58a19ddb:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731751082172.d9e28756f78b0555a6733ffbf6ab98bd. 2024-11-16T09:58:04,999 ERROR [RS_FLUSH_OPERATIONS-regionserver/037c58a19ddb:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] handler.RSProcedureHandler(58): pid=8 java.io.IOException: Unable to complete flush {ENCODED => d9e28756f78b0555a6733ffbf6ab98bd, NAME => 'TestAcidGuarantees,,1731751082172.d9e28756f78b0555a6733ffbf6ab98bd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T09:58:04,999 DEBUG [RS_FLUSH_OPERATIONS-regionserver/037c58a19ddb:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=8 java.io.IOException: Unable to complete flush {ENCODED => d9e28756f78b0555a6733ffbf6ab98bd, NAME => 'TestAcidGuarantees,,1731751082172.d9e28756f78b0555a6733ffbf6ab98bd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T09:58:05,000 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~158.91 KB/162720, currentSize=154.31 KB/158010 for d9e28756f78b0555a6733ffbf6ab98bd in 469ms, sequenceid=13, compaction requested=false 2024-11-16T09:58:05,001 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37877 {}] master.HMaster(4177): Remote procedure failed, pid=8 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d9e28756f78b0555a6733ffbf6ab98bd, NAME => 'TestAcidGuarantees,,1731751082172.d9e28756f78b0555a6733ffbf6ab98bd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d9e28756f78b0555a6733ffbf6ab98bd, NAME => 'TestAcidGuarantees,,1731751082172.d9e28756f78b0555a6733ffbf6ab98bd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T09:58:05,002 DEBUG [MemStoreFlusher.0 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'TestAcidGuarantees' 2024-11-16T09:58:05,008 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for d9e28756f78b0555a6733ffbf6ab98bd: 2024-11-16T09:58:05,028 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-16T09:58:05,029 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing d9e28756f78b0555a6733ffbf6ab98bd 3/3 column families, dataSize=161.02 KB heapSize=422.63 KB 2024-11-16T09:58:05,029 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] regionserver.HRegion(8855): Flush requested on d9e28756f78b0555a6733ffbf6ab98bd 2024-11-16T09:58:05,042 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/d9e28756f78b0555a6733ffbf6ab98bd/.tmp/A/263ffc1427114090a4723fb2b1539548 is 50, key is test_row_0/A:col10/1731751084693/Put/seqid=0 2024-11-16T09:58:05,066 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37591 is added to blk_1073741840_1016 (size=14341) 2024-11-16T09:58:05,068 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=53.67 KB at sequenceid=40 (bloomFilter=true), to=hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/d9e28756f78b0555a6733ffbf6ab98bd/.tmp/A/263ffc1427114090a4723fb2b1539548 2024-11-16T09:58:05,076 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9e28756f78b0555a6733ffbf6ab98bd, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T09:58:05,076 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39620 deadline: 1731751145061, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9e28756f78b0555a6733ffbf6ab98bd, server=037c58a19ddb,37267,1731751077844 2024-11-16T09:58:05,077 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731751082172.d9e28756f78b0555a6733ffbf6ab98bd., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 , the old value is region=TestAcidGuarantees,,1731751082172.d9e28756f78b0555a6733ffbf6ab98bd., hostname=037c58a19ddb,37267,1731751077844, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9e28756f78b0555a6733ffbf6ab98bd, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:58:05,077 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731751082172.d9e28756f78b0555a6733ffbf6ab98bd., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9e28756f78b0555a6733ffbf6ab98bd, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:58:05,077 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9e28756f78b0555a6733ffbf6ab98bd, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T09:58:05,077 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731751082172.d9e28756f78b0555a6733ffbf6ab98bd., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 because the exception is null or not the one we care about 2024-11-16T09:58:05,078 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39638 deadline: 1731751145063, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9e28756f78b0555a6733ffbf6ab98bd, server=037c58a19ddb,37267,1731751077844 2024-11-16T09:58:05,079 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731751082172.d9e28756f78b0555a6733ffbf6ab98bd., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 , the old value is region=TestAcidGuarantees,,1731751082172.d9e28756f78b0555a6733ffbf6ab98bd., hostname=037c58a19ddb,37267,1731751077844, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9e28756f78b0555a6733ffbf6ab98bd, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:58:05,079 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731751082172.d9e28756f78b0555a6733ffbf6ab98bd., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9e28756f78b0555a6733ffbf6ab98bd, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:58:05,079 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731751082172.d9e28756f78b0555a6733ffbf6ab98bd., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 because the exception is null or not the one we care about 2024-11-16T09:58:05,079 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9e28756f78b0555a6733ffbf6ab98bd, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T09:58:05,079 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39696 deadline: 1731751145064, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9e28756f78b0555a6733ffbf6ab98bd, server=037c58a19ddb,37267,1731751077844 2024-11-16T09:58:05,079 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731751082172.d9e28756f78b0555a6733ffbf6ab98bd., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 , the old value is region=TestAcidGuarantees,,1731751082172.d9e28756f78b0555a6733ffbf6ab98bd., hostname=037c58a19ddb,37267,1731751077844, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9e28756f78b0555a6733ffbf6ab98bd, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:58:05,079 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731751082172.d9e28756f78b0555a6733ffbf6ab98bd., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9e28756f78b0555a6733ffbf6ab98bd, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:58:05,080 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731751082172.d9e28756f78b0555a6733ffbf6ab98bd., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 because the exception is null or not the one we care about 2024-11-16T09:58:05,080 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9e28756f78b0555a6733ffbf6ab98bd, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T09:58:05,080 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39630 deadline: 1731751145068, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9e28756f78b0555a6733ffbf6ab98bd, server=037c58a19ddb,37267,1731751077844 2024-11-16T09:58:05,081 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731751082172.d9e28756f78b0555a6733ffbf6ab98bd., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 , the old value is region=TestAcidGuarantees,,1731751082172.d9e28756f78b0555a6733ffbf6ab98bd., hostname=037c58a19ddb,37267,1731751077844, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9e28756f78b0555a6733ffbf6ab98bd, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:58:05,081 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731751082172.d9e28756f78b0555a6733ffbf6ab98bd., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9e28756f78b0555a6733ffbf6ab98bd, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:58:05,081 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731751082172.d9e28756f78b0555a6733ffbf6ab98bd., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 because the exception is null or not the one we care about 2024-11-16T09:58:05,089 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/d9e28756f78b0555a6733ffbf6ab98bd/.tmp/B/c21d5ad86bf84c7f9d79b1232a65de8b is 50, key is test_row_0/B:col10/1731751084693/Put/seqid=0 2024-11-16T09:58:05,093 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9e28756f78b0555a6733ffbf6ab98bd, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T09:58:05,094 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39654 deadline: 1731751145083, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9e28756f78b0555a6733ffbf6ab98bd, server=037c58a19ddb,37267,1731751077844 2024-11-16T09:58:05,095 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731751082172.d9e28756f78b0555a6733ffbf6ab98bd., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 , the old value is region=TestAcidGuarantees,,1731751082172.d9e28756f78b0555a6733ffbf6ab98bd., hostname=037c58a19ddb,37267,1731751077844, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9e28756f78b0555a6733ffbf6ab98bd, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:58:05,095 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37877 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=7 2024-11-16T09:58:05,095 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731751082172.d9e28756f78b0555a6733ffbf6ab98bd., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9e28756f78b0555a6733ffbf6ab98bd, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:58:05,095 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731751082172.d9e28756f78b0555a6733ffbf6ab98bd., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 because the exception is null or not the one we care about 2024-11-16T09:58:05,110 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37591 is added to blk_1073741841_1017 (size=12001) 2024-11-16T09:58:05,112 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=53.67 KB at sequenceid=40 (bloomFilter=true), to=hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/d9e28756f78b0555a6733ffbf6ab98bd/.tmp/B/c21d5ad86bf84c7f9d79b1232a65de8b 2024-11-16T09:58:05,137 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/d9e28756f78b0555a6733ffbf6ab98bd/.tmp/C/1537924938324874ace183a7fc8c33fb is 50, key is test_row_0/C:col10/1731751084693/Put/seqid=0 2024-11-16T09:58:05,156 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=37267 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=8 2024-11-16T09:58:05,156 DEBUG [RS_FLUSH_OPERATIONS-regionserver/037c58a19ddb:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731751082172.d9e28756f78b0555a6733ffbf6ab98bd. 2024-11-16T09:58:05,157 DEBUG [RS_FLUSH_OPERATIONS-regionserver/037c58a19ddb:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731751082172.d9e28756f78b0555a6733ffbf6ab98bd. as already flushing 2024-11-16T09:58:05,157 DEBUG [RS_FLUSH_OPERATIONS-regionserver/037c58a19ddb:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731751082172.d9e28756f78b0555a6733ffbf6ab98bd. 2024-11-16T09:58:05,157 ERROR [RS_FLUSH_OPERATIONS-regionserver/037c58a19ddb:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] handler.RSProcedureHandler(58): pid=8 java.io.IOException: Unable to complete flush {ENCODED => d9e28756f78b0555a6733ffbf6ab98bd, NAME => 'TestAcidGuarantees,,1731751082172.d9e28756f78b0555a6733ffbf6ab98bd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T09:58:05,157 DEBUG [RS_FLUSH_OPERATIONS-regionserver/037c58a19ddb:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=8 java.io.IOException: Unable to complete flush {ENCODED => d9e28756f78b0555a6733ffbf6ab98bd, NAME => 'TestAcidGuarantees,,1731751082172.d9e28756f78b0555a6733ffbf6ab98bd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T09:58:05,159 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37877 {}] master.HMaster(4177): Remote procedure failed, pid=8 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d9e28756f78b0555a6733ffbf6ab98bd, NAME => 'TestAcidGuarantees,,1731751082172.d9e28756f78b0555a6733ffbf6ab98bd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d9e28756f78b0555a6733ffbf6ab98bd, NAME => 'TestAcidGuarantees,,1731751082172.d9e28756f78b0555a6733ffbf6ab98bd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T09:58:05,174 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37591 is added to blk_1073741842_1018 (size=12001) 2024-11-16T09:58:05,194 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9e28756f78b0555a6733ffbf6ab98bd, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T09:58:05,194 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39620 deadline: 1731751145184, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9e28756f78b0555a6733ffbf6ab98bd, server=037c58a19ddb,37267,1731751077844 2024-11-16T09:58:05,195 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731751082172.d9e28756f78b0555a6733ffbf6ab98bd., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 , the old value is region=TestAcidGuarantees,,1731751082172.d9e28756f78b0555a6733ffbf6ab98bd., hostname=037c58a19ddb,37267,1731751077844, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9e28756f78b0555a6733ffbf6ab98bd, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:58:05,195 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9e28756f78b0555a6733ffbf6ab98bd, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T09:58:05,195 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731751082172.d9e28756f78b0555a6733ffbf6ab98bd., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9e28756f78b0555a6733ffbf6ab98bd, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:58:05,195 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731751082172.d9e28756f78b0555a6733ffbf6ab98bd., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 because the exception is null or not the one we care about 2024-11-16T09:58:05,196 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39638 deadline: 1731751145185, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9e28756f78b0555a6733ffbf6ab98bd, server=037c58a19ddb,37267,1731751077844 2024-11-16T09:58:05,196 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731751082172.d9e28756f78b0555a6733ffbf6ab98bd., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 , the old value is region=TestAcidGuarantees,,1731751082172.d9e28756f78b0555a6733ffbf6ab98bd., hostname=037c58a19ddb,37267,1731751077844, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9e28756f78b0555a6733ffbf6ab98bd, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:58:05,196 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731751082172.d9e28756f78b0555a6733ffbf6ab98bd., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9e28756f78b0555a6733ffbf6ab98bd, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:58:05,197 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731751082172.d9e28756f78b0555a6733ffbf6ab98bd., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 because the exception is null or not the one we care about 2024-11-16T09:58:05,196 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9e28756f78b0555a6733ffbf6ab98bd, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T09:58:05,197 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39696 deadline: 1731751145185, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9e28756f78b0555a6733ffbf6ab98bd, server=037c58a19ddb,37267,1731751077844 2024-11-16T09:58:05,197 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731751082172.d9e28756f78b0555a6733ffbf6ab98bd., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 , the old value is region=TestAcidGuarantees,,1731751082172.d9e28756f78b0555a6733ffbf6ab98bd., hostname=037c58a19ddb,37267,1731751077844, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9e28756f78b0555a6733ffbf6ab98bd, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:58:05,198 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731751082172.d9e28756f78b0555a6733ffbf6ab98bd., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9e28756f78b0555a6733ffbf6ab98bd, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:58:05,198 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731751082172.d9e28756f78b0555a6733ffbf6ab98bd., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 because the exception is null or not the one we care about 2024-11-16T09:58:05,204 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9e28756f78b0555a6733ffbf6ab98bd, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T09:58:05,204 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39630 deadline: 1731751145195, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9e28756f78b0555a6733ffbf6ab98bd, server=037c58a19ddb,37267,1731751077844 2024-11-16T09:58:05,205 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731751082172.d9e28756f78b0555a6733ffbf6ab98bd., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 , the old value is region=TestAcidGuarantees,,1731751082172.d9e28756f78b0555a6733ffbf6ab98bd., hostname=037c58a19ddb,37267,1731751077844, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9e28756f78b0555a6733ffbf6ab98bd, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:58:05,205 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731751082172.d9e28756f78b0555a6733ffbf6ab98bd., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9e28756f78b0555a6733ffbf6ab98bd, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:58:05,205 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731751082172.d9e28756f78b0555a6733ffbf6ab98bd., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 because the exception is null or not the one we care about 2024-11-16T09:58:05,209 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9e28756f78b0555a6733ffbf6ab98bd, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T09:58:05,210 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39654 deadline: 1731751145204, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9e28756f78b0555a6733ffbf6ab98bd, server=037c58a19ddb,37267,1731751077844 2024-11-16T09:58:05,210 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731751082172.d9e28756f78b0555a6733ffbf6ab98bd., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 , the old value is region=TestAcidGuarantees,,1731751082172.d9e28756f78b0555a6733ffbf6ab98bd., hostname=037c58a19ddb,37267,1731751077844, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9e28756f78b0555a6733ffbf6ab98bd, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:58:05,211 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731751082172.d9e28756f78b0555a6733ffbf6ab98bd., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9e28756f78b0555a6733ffbf6ab98bd, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:58:05,211 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731751082172.d9e28756f78b0555a6733ffbf6ab98bd., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 because the exception is null or not the one we care about 2024-11-16T09:58:05,313 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=37267 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=8 2024-11-16T09:58:05,313 DEBUG [RS_FLUSH_OPERATIONS-regionserver/037c58a19ddb:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731751082172.d9e28756f78b0555a6733ffbf6ab98bd. 2024-11-16T09:58:05,313 DEBUG [RS_FLUSH_OPERATIONS-regionserver/037c58a19ddb:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731751082172.d9e28756f78b0555a6733ffbf6ab98bd. as already flushing 2024-11-16T09:58:05,314 DEBUG [RS_FLUSH_OPERATIONS-regionserver/037c58a19ddb:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731751082172.d9e28756f78b0555a6733ffbf6ab98bd. 2024-11-16T09:58:05,314 ERROR [RS_FLUSH_OPERATIONS-regionserver/037c58a19ddb:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] handler.RSProcedureHandler(58): pid=8 java.io.IOException: Unable to complete flush {ENCODED => d9e28756f78b0555a6733ffbf6ab98bd, NAME => 'TestAcidGuarantees,,1731751082172.d9e28756f78b0555a6733ffbf6ab98bd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T09:58:05,314 DEBUG [RS_FLUSH_OPERATIONS-regionserver/037c58a19ddb:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=8 java.io.IOException: Unable to complete flush {ENCODED => d9e28756f78b0555a6733ffbf6ab98bd, NAME => 'TestAcidGuarantees,,1731751082172.d9e28756f78b0555a6733ffbf6ab98bd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T09:58:05,315 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37877 {}] master.HMaster(4177): Remote procedure failed, pid=8 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d9e28756f78b0555a6733ffbf6ab98bd, NAME => 'TestAcidGuarantees,,1731751082172.d9e28756f78b0555a6733ffbf6ab98bd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d9e28756f78b0555a6733ffbf6ab98bd, NAME => 'TestAcidGuarantees,,1731751082172.d9e28756f78b0555a6733ffbf6ab98bd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T09:58:05,405 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9e28756f78b0555a6733ffbf6ab98bd, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T09:58:05,406 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39620 deadline: 1731751145403, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9e28756f78b0555a6733ffbf6ab98bd, server=037c58a19ddb,37267,1731751077844 2024-11-16T09:58:05,407 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731751082172.d9e28756f78b0555a6733ffbf6ab98bd., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 , the old value is region=TestAcidGuarantees,,1731751082172.d9e28756f78b0555a6733ffbf6ab98bd., hostname=037c58a19ddb,37267,1731751077844, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9e28756f78b0555a6733ffbf6ab98bd, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:58:05,407 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731751082172.d9e28756f78b0555a6733ffbf6ab98bd., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9e28756f78b0555a6733ffbf6ab98bd, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:58:05,407 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731751082172.d9e28756f78b0555a6733ffbf6ab98bd., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 because the exception is null or not the one we care about 2024-11-16T09:58:05,407 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9e28756f78b0555a6733ffbf6ab98bd, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T09:58:05,408 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39696 deadline: 1731751145404, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9e28756f78b0555a6733ffbf6ab98bd, server=037c58a19ddb,37267,1731751077844 2024-11-16T09:58:05,408 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9e28756f78b0555a6733ffbf6ab98bd, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T09:58:05,408 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39638 deadline: 1731751145404, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9e28756f78b0555a6733ffbf6ab98bd, server=037c58a19ddb,37267,1731751077844 2024-11-16T09:58:05,409 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731751082172.d9e28756f78b0555a6733ffbf6ab98bd., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 , the old value is region=TestAcidGuarantees,,1731751082172.d9e28756f78b0555a6733ffbf6ab98bd., hostname=037c58a19ddb,37267,1731751077844, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9e28756f78b0555a6733ffbf6ab98bd, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:58:05,409 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731751082172.d9e28756f78b0555a6733ffbf6ab98bd., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 , the old value is region=TestAcidGuarantees,,1731751082172.d9e28756f78b0555a6733ffbf6ab98bd., hostname=037c58a19ddb,37267,1731751077844, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9e28756f78b0555a6733ffbf6ab98bd, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:58:05,409 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731751082172.d9e28756f78b0555a6733ffbf6ab98bd., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9e28756f78b0555a6733ffbf6ab98bd, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:58:05,409 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731751082172.d9e28756f78b0555a6733ffbf6ab98bd., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9e28756f78b0555a6733ffbf6ab98bd, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:58:05,409 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731751082172.d9e28756f78b0555a6733ffbf6ab98bd., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 because the exception is null or not the one we care about 2024-11-16T09:58:05,409 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731751082172.d9e28756f78b0555a6733ffbf6ab98bd., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 because the exception is null or not the one we care about 2024-11-16T09:58:05,421 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9e28756f78b0555a6733ffbf6ab98bd, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T09:58:05,422 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39630 deadline: 1731751145414, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9e28756f78b0555a6733ffbf6ab98bd, server=037c58a19ddb,37267,1731751077844 2024-11-16T09:58:05,423 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731751082172.d9e28756f78b0555a6733ffbf6ab98bd., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 , the old value is region=TestAcidGuarantees,,1731751082172.d9e28756f78b0555a6733ffbf6ab98bd., hostname=037c58a19ddb,37267,1731751077844, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9e28756f78b0555a6733ffbf6ab98bd, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:58:05,423 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731751082172.d9e28756f78b0555a6733ffbf6ab98bd., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9e28756f78b0555a6733ffbf6ab98bd, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:58:05,423 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731751082172.d9e28756f78b0555a6733ffbf6ab98bd., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 because the exception is null or not the one we care about 2024-11-16T09:58:05,425 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9e28756f78b0555a6733ffbf6ab98bd, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T09:58:05,426 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39654 deadline: 1731751145424, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9e28756f78b0555a6733ffbf6ab98bd, server=037c58a19ddb,37267,1731751077844 2024-11-16T09:58:05,426 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731751082172.d9e28756f78b0555a6733ffbf6ab98bd., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 , the old value is region=TestAcidGuarantees,,1731751082172.d9e28756f78b0555a6733ffbf6ab98bd., hostname=037c58a19ddb,37267,1731751077844, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9e28756f78b0555a6733ffbf6ab98bd, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:58:05,426 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731751082172.d9e28756f78b0555a6733ffbf6ab98bd., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9e28756f78b0555a6733ffbf6ab98bd, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:58:05,427 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731751082172.d9e28756f78b0555a6733ffbf6ab98bd., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 because the exception is null or not the one we care about 2024-11-16T09:58:05,470 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=37267 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=8 2024-11-16T09:58:05,470 DEBUG [RS_FLUSH_OPERATIONS-regionserver/037c58a19ddb:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731751082172.d9e28756f78b0555a6733ffbf6ab98bd. 2024-11-16T09:58:05,470 DEBUG [RS_FLUSH_OPERATIONS-regionserver/037c58a19ddb:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731751082172.d9e28756f78b0555a6733ffbf6ab98bd. as already flushing 2024-11-16T09:58:05,470 DEBUG [RS_FLUSH_OPERATIONS-regionserver/037c58a19ddb:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731751082172.d9e28756f78b0555a6733ffbf6ab98bd. 2024-11-16T09:58:05,471 ERROR [RS_FLUSH_OPERATIONS-regionserver/037c58a19ddb:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] handler.RSProcedureHandler(58): pid=8 java.io.IOException: Unable to complete flush {ENCODED => d9e28756f78b0555a6733ffbf6ab98bd, NAME => 'TestAcidGuarantees,,1731751082172.d9e28756f78b0555a6733ffbf6ab98bd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T09:58:05,471 DEBUG [RS_FLUSH_OPERATIONS-regionserver/037c58a19ddb:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=8 java.io.IOException: Unable to complete flush {ENCODED => d9e28756f78b0555a6733ffbf6ab98bd, NAME => 'TestAcidGuarantees,,1731751082172.d9e28756f78b0555a6733ffbf6ab98bd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T09:58:05,472 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37877 {}] master.HMaster(4177): Remote procedure failed, pid=8 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d9e28756f78b0555a6733ffbf6ab98bd, NAME => 'TestAcidGuarantees,,1731751082172.d9e28756f78b0555a6733ffbf6ab98bd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d9e28756f78b0555a6733ffbf6ab98bd, NAME => 'TestAcidGuarantees,,1731751082172.d9e28756f78b0555a6733ffbf6ab98bd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T09:58:05,568 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=53.67 KB at sequenceid=40 (bloomFilter=true), to=hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/d9e28756f78b0555a6733ffbf6ab98bd/.tmp/C/1537924938324874ace183a7fc8c33fb 2024-11-16T09:58:05,587 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/d9e28756f78b0555a6733ffbf6ab98bd/.tmp/A/263ffc1427114090a4723fb2b1539548 as hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/d9e28756f78b0555a6733ffbf6ab98bd/A/263ffc1427114090a4723fb2b1539548 2024-11-16T09:58:05,601 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/d9e28756f78b0555a6733ffbf6ab98bd/A/263ffc1427114090a4723fb2b1539548, entries=200, sequenceid=40, filesize=14.0 K 2024-11-16T09:58:05,604 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37877 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=7 2024-11-16T09:58:05,605 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/d9e28756f78b0555a6733ffbf6ab98bd/.tmp/B/c21d5ad86bf84c7f9d79b1232a65de8b as hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/d9e28756f78b0555a6733ffbf6ab98bd/B/c21d5ad86bf84c7f9d79b1232a65de8b 2024-11-16T09:58:05,617 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/d9e28756f78b0555a6733ffbf6ab98bd/B/c21d5ad86bf84c7f9d79b1232a65de8b, entries=150, sequenceid=40, filesize=11.7 K 2024-11-16T09:58:05,619 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/d9e28756f78b0555a6733ffbf6ab98bd/.tmp/C/1537924938324874ace183a7fc8c33fb as hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/d9e28756f78b0555a6733ffbf6ab98bd/C/1537924938324874ace183a7fc8c33fb 2024-11-16T09:58:05,625 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=37267 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=8 2024-11-16T09:58:05,626 DEBUG [RS_FLUSH_OPERATIONS-regionserver/037c58a19ddb:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731751082172.d9e28756f78b0555a6733ffbf6ab98bd. 2024-11-16T09:58:05,626 DEBUG [RS_FLUSH_OPERATIONS-regionserver/037c58a19ddb:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731751082172.d9e28756f78b0555a6733ffbf6ab98bd. as already flushing 2024-11-16T09:58:05,626 DEBUG [RS_FLUSH_OPERATIONS-regionserver/037c58a19ddb:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731751082172.d9e28756f78b0555a6733ffbf6ab98bd. 2024-11-16T09:58:05,626 ERROR [RS_FLUSH_OPERATIONS-regionserver/037c58a19ddb:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] handler.RSProcedureHandler(58): pid=8 java.io.IOException: Unable to complete flush {ENCODED => d9e28756f78b0555a6733ffbf6ab98bd, NAME => 'TestAcidGuarantees,,1731751082172.d9e28756f78b0555a6733ffbf6ab98bd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T09:58:05,627 DEBUG [RS_FLUSH_OPERATIONS-regionserver/037c58a19ddb:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=8 java.io.IOException: Unable to complete flush {ENCODED => d9e28756f78b0555a6733ffbf6ab98bd, NAME => 'TestAcidGuarantees,,1731751082172.d9e28756f78b0555a6733ffbf6ab98bd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T09:58:05,628 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37877 {}] master.HMaster(4177): Remote procedure failed, pid=8 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d9e28756f78b0555a6733ffbf6ab98bd, NAME => 'TestAcidGuarantees,,1731751082172.d9e28756f78b0555a6733ffbf6ab98bd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d9e28756f78b0555a6733ffbf6ab98bd, NAME => 'TestAcidGuarantees,,1731751082172.d9e28756f78b0555a6733ffbf6ab98bd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T09:58:05,636 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/d9e28756f78b0555a6733ffbf6ab98bd/C/1537924938324874ace183a7fc8c33fb, entries=150, sequenceid=40, filesize=11.7 K 2024-11-16T09:58:05,640 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~161.02 KB/164880, heapSize ~422.58 KB/432720, currentSize=53.67 KB/54960 for d9e28756f78b0555a6733ffbf6ab98bd in 612ms, sequenceid=40, compaction requested=false 2024-11-16T09:58:05,641 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for d9e28756f78b0555a6733ffbf6ab98bd: 2024-11-16T09:58:05,721 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] regionserver.HRegion(8855): Flush requested on d9e28756f78b0555a6733ffbf6ab98bd 2024-11-16T09:58:05,721 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-16T09:58:05,722 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing d9e28756f78b0555a6733ffbf6ab98bd 3/3 column families, dataSize=60.38 KB heapSize=158.95 KB 2024-11-16T09:58:05,743 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/d9e28756f78b0555a6733ffbf6ab98bd/.tmp/A/fd8aea0564294c4d927efa1b14d23126 is 50, key is test_row_0/A:col10/1731751085061/Put/seqid=0 2024-11-16T09:58:05,770 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37591 is added to blk_1073741843_1019 (size=12001) 2024-11-16T09:58:05,771 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=53 (bloomFilter=true), to=hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/d9e28756f78b0555a6733ffbf6ab98bd/.tmp/A/fd8aea0564294c4d927efa1b14d23126 2024-11-16T09:58:05,782 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=37267 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=8 2024-11-16T09:58:05,783 DEBUG [RS_FLUSH_OPERATIONS-regionserver/037c58a19ddb:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731751082172.d9e28756f78b0555a6733ffbf6ab98bd. 2024-11-16T09:58:05,783 DEBUG [RS_FLUSH_OPERATIONS-regionserver/037c58a19ddb:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731751082172.d9e28756f78b0555a6733ffbf6ab98bd. as already flushing 2024-11-16T09:58:05,783 DEBUG [RS_FLUSH_OPERATIONS-regionserver/037c58a19ddb:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731751082172.d9e28756f78b0555a6733ffbf6ab98bd. 2024-11-16T09:58:05,783 ERROR [RS_FLUSH_OPERATIONS-regionserver/037c58a19ddb:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] handler.RSProcedureHandler(58): pid=8 java.io.IOException: Unable to complete flush {ENCODED => d9e28756f78b0555a6733ffbf6ab98bd, NAME => 'TestAcidGuarantees,,1731751082172.d9e28756f78b0555a6733ffbf6ab98bd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T09:58:05,783 DEBUG [RS_FLUSH_OPERATIONS-regionserver/037c58a19ddb:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=8 java.io.IOException: Unable to complete flush {ENCODED => d9e28756f78b0555a6733ffbf6ab98bd, NAME => 'TestAcidGuarantees,,1731751082172.d9e28756f78b0555a6733ffbf6ab98bd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T09:58:05,785 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37877 {}] master.HMaster(4177): Remote procedure failed, pid=8 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d9e28756f78b0555a6733ffbf6ab98bd, NAME => 'TestAcidGuarantees,,1731751082172.d9e28756f78b0555a6733ffbf6ab98bd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d9e28756f78b0555a6733ffbf6ab98bd, NAME => 'TestAcidGuarantees,,1731751082172.d9e28756f78b0555a6733ffbf6ab98bd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T09:58:05,799 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9e28756f78b0555a6733ffbf6ab98bd, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T09:58:05,800 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/d9e28756f78b0555a6733ffbf6ab98bd/.tmp/B/983f5ee278db4f3eaac85d7f97591fda is 50, key is test_row_0/B:col10/1731751085061/Put/seqid=0 2024-11-16T09:58:05,800 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39630 deadline: 1731751145789, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9e28756f78b0555a6733ffbf6ab98bd, server=037c58a19ddb,37267,1731751077844 2024-11-16T09:58:05,801 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731751082172.d9e28756f78b0555a6733ffbf6ab98bd., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 , the old value is region=TestAcidGuarantees,,1731751082172.d9e28756f78b0555a6733ffbf6ab98bd., hostname=037c58a19ddb,37267,1731751077844, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9e28756f78b0555a6733ffbf6ab98bd, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:58:05,801 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731751082172.d9e28756f78b0555a6733ffbf6ab98bd., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9e28756f78b0555a6733ffbf6ab98bd, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:58:05,801 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731751082172.d9e28756f78b0555a6733ffbf6ab98bd., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 because the exception is null or not the one we care about 2024-11-16T09:58:05,801 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9e28756f78b0555a6733ffbf6ab98bd, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T09:58:05,801 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39620 deadline: 1731751145789, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9e28756f78b0555a6733ffbf6ab98bd, server=037c58a19ddb,37267,1731751077844 2024-11-16T09:58:05,802 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731751082172.d9e28756f78b0555a6733ffbf6ab98bd., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 , the old value is region=TestAcidGuarantees,,1731751082172.d9e28756f78b0555a6733ffbf6ab98bd., hostname=037c58a19ddb,37267,1731751077844, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9e28756f78b0555a6733ffbf6ab98bd, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:58:05,802 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731751082172.d9e28756f78b0555a6733ffbf6ab98bd., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9e28756f78b0555a6733ffbf6ab98bd, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:58:05,802 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731751082172.d9e28756f78b0555a6733ffbf6ab98bd., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 because the exception is null or not the one we care about 2024-11-16T09:58:05,803 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9e28756f78b0555a6733ffbf6ab98bd, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T09:58:05,804 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39638 deadline: 1731751145794, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9e28756f78b0555a6733ffbf6ab98bd, server=037c58a19ddb,37267,1731751077844 2024-11-16T09:58:05,804 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9e28756f78b0555a6733ffbf6ab98bd, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T09:58:05,804 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39654 deadline: 1731751145795, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9e28756f78b0555a6733ffbf6ab98bd, server=037c58a19ddb,37267,1731751077844 2024-11-16T09:58:05,805 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731751082172.d9e28756f78b0555a6733ffbf6ab98bd., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 , the old value is region=TestAcidGuarantees,,1731751082172.d9e28756f78b0555a6733ffbf6ab98bd., hostname=037c58a19ddb,37267,1731751077844, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9e28756f78b0555a6733ffbf6ab98bd, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:58:05,805 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731751082172.d9e28756f78b0555a6733ffbf6ab98bd., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9e28756f78b0555a6733ffbf6ab98bd, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:58:05,805 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731751082172.d9e28756f78b0555a6733ffbf6ab98bd., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 because the exception is null or not the one we care about 2024-11-16T09:58:05,805 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731751082172.d9e28756f78b0555a6733ffbf6ab98bd., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 , the old value is region=TestAcidGuarantees,,1731751082172.d9e28756f78b0555a6733ffbf6ab98bd., hostname=037c58a19ddb,37267,1731751077844, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9e28756f78b0555a6733ffbf6ab98bd, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:58:05,805 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731751082172.d9e28756f78b0555a6733ffbf6ab98bd., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9e28756f78b0555a6733ffbf6ab98bd, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:58:05,805 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9e28756f78b0555a6733ffbf6ab98bd, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T09:58:05,805 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731751082172.d9e28756f78b0555a6733ffbf6ab98bd., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 because the exception is null or not the one we care about 2024-11-16T09:58:05,805 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39696 deadline: 1731751145797, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9e28756f78b0555a6733ffbf6ab98bd, server=037c58a19ddb,37267,1731751077844 2024-11-16T09:58:05,806 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731751082172.d9e28756f78b0555a6733ffbf6ab98bd., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 , the old value is region=TestAcidGuarantees,,1731751082172.d9e28756f78b0555a6733ffbf6ab98bd., hostname=037c58a19ddb,37267,1731751077844, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9e28756f78b0555a6733ffbf6ab98bd, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:58:05,806 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731751082172.d9e28756f78b0555a6733ffbf6ab98bd., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9e28756f78b0555a6733ffbf6ab98bd, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:58:05,806 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731751082172.d9e28756f78b0555a6733ffbf6ab98bd., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 because the exception is null or not the one we care about 2024-11-16T09:58:05,818 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37591 is added to blk_1073741844_1020 (size=12001) 2024-11-16T09:58:05,820 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=53 (bloomFilter=true), to=hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/d9e28756f78b0555a6733ffbf6ab98bd/.tmp/B/983f5ee278db4f3eaac85d7f97591fda 2024-11-16T09:58:05,843 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/d9e28756f78b0555a6733ffbf6ab98bd/.tmp/C/53be78a7cd2a4f1d98224b1c2affc854 is 50, key is test_row_0/C:col10/1731751085061/Put/seqid=0 2024-11-16T09:58:05,860 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37591 is added to blk_1073741845_1021 (size=12001) 2024-11-16T09:58:05,861 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=53 (bloomFilter=true), to=hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/d9e28756f78b0555a6733ffbf6ab98bd/.tmp/C/53be78a7cd2a4f1d98224b1c2affc854 2024-11-16T09:58:05,875 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/d9e28756f78b0555a6733ffbf6ab98bd/.tmp/A/fd8aea0564294c4d927efa1b14d23126 as hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/d9e28756f78b0555a6733ffbf6ab98bd/A/fd8aea0564294c4d927efa1b14d23126 2024-11-16T09:58:05,890 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/d9e28756f78b0555a6733ffbf6ab98bd/A/fd8aea0564294c4d927efa1b14d23126, entries=150, sequenceid=53, filesize=11.7 K 2024-11-16T09:58:05,892 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/d9e28756f78b0555a6733ffbf6ab98bd/.tmp/B/983f5ee278db4f3eaac85d7f97591fda as hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/d9e28756f78b0555a6733ffbf6ab98bd/B/983f5ee278db4f3eaac85d7f97591fda 2024-11-16T09:58:05,904 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9e28756f78b0555a6733ffbf6ab98bd, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T09:58:05,905 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39630 deadline: 1731751145903, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9e28756f78b0555a6733ffbf6ab98bd, server=037c58a19ddb,37267,1731751077844 2024-11-16T09:58:05,906 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731751082172.d9e28756f78b0555a6733ffbf6ab98bd., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 , the old value is region=TestAcidGuarantees,,1731751082172.d9e28756f78b0555a6733ffbf6ab98bd., hostname=037c58a19ddb,37267,1731751077844, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9e28756f78b0555a6733ffbf6ab98bd, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:58:05,906 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731751082172.d9e28756f78b0555a6733ffbf6ab98bd., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9e28756f78b0555a6733ffbf6ab98bd, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:58:05,906 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731751082172.d9e28756f78b0555a6733ffbf6ab98bd., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 because the exception is null or not the one we care about 2024-11-16T09:58:05,908 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/d9e28756f78b0555a6733ffbf6ab98bd/B/983f5ee278db4f3eaac85d7f97591fda, entries=150, sequenceid=53, filesize=11.7 K 2024-11-16T09:58:05,910 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/d9e28756f78b0555a6733ffbf6ab98bd/.tmp/C/53be78a7cd2a4f1d98224b1c2affc854 as hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/d9e28756f78b0555a6733ffbf6ab98bd/C/53be78a7cd2a4f1d98224b1c2affc854 2024-11-16T09:58:05,917 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9e28756f78b0555a6733ffbf6ab98bd, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T09:58:05,918 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39620 deadline: 1731751145914, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9e28756f78b0555a6733ffbf6ab98bd, server=037c58a19ddb,37267,1731751077844 2024-11-16T09:58:05,918 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731751082172.d9e28756f78b0555a6733ffbf6ab98bd., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 , the old value is region=TestAcidGuarantees,,1731751082172.d9e28756f78b0555a6733ffbf6ab98bd., hostname=037c58a19ddb,37267,1731751077844, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9e28756f78b0555a6733ffbf6ab98bd, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:58:05,919 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731751082172.d9e28756f78b0555a6733ffbf6ab98bd., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9e28756f78b0555a6733ffbf6ab98bd, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:58:05,919 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731751082172.d9e28756f78b0555a6733ffbf6ab98bd., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 because the exception is null or not the one we care about 2024-11-16T09:58:05,919 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9e28756f78b0555a6733ffbf6ab98bd, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T09:58:05,919 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9e28756f78b0555a6733ffbf6ab98bd, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T09:58:05,919 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39638 deadline: 1731751145914, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9e28756f78b0555a6733ffbf6ab98bd, server=037c58a19ddb,37267,1731751077844 2024-11-16T09:58:05,919 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39654 deadline: 1731751145918, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9e28756f78b0555a6733ffbf6ab98bd, server=037c58a19ddb,37267,1731751077844 2024-11-16T09:58:05,919 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9e28756f78b0555a6733ffbf6ab98bd, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T09:58:05,920 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731751082172.d9e28756f78b0555a6733ffbf6ab98bd., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 , the old value is region=TestAcidGuarantees,,1731751082172.d9e28756f78b0555a6733ffbf6ab98bd., hostname=037c58a19ddb,37267,1731751077844, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9e28756f78b0555a6733ffbf6ab98bd, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:58:05,920 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39696 deadline: 1731751145918, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9e28756f78b0555a6733ffbf6ab98bd, server=037c58a19ddb,37267,1731751077844 2024-11-16T09:58:05,920 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731751082172.d9e28756f78b0555a6733ffbf6ab98bd., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9e28756f78b0555a6733ffbf6ab98bd, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:58:05,920 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731751082172.d9e28756f78b0555a6733ffbf6ab98bd., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 because the exception is null or not the one we care about 2024-11-16T09:58:05,920 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731751082172.d9e28756f78b0555a6733ffbf6ab98bd., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 , the old value is region=TestAcidGuarantees,,1731751082172.d9e28756f78b0555a6733ffbf6ab98bd., hostname=037c58a19ddb,37267,1731751077844, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9e28756f78b0555a6733ffbf6ab98bd, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:58:05,921 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731751082172.d9e28756f78b0555a6733ffbf6ab98bd., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9e28756f78b0555a6733ffbf6ab98bd, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:58:05,921 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731751082172.d9e28756f78b0555a6733ffbf6ab98bd., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 because the exception is null or not the one we care about 2024-11-16T09:58:05,921 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731751082172.d9e28756f78b0555a6733ffbf6ab98bd., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 , the old value is region=TestAcidGuarantees,,1731751082172.d9e28756f78b0555a6733ffbf6ab98bd., hostname=037c58a19ddb,37267,1731751077844, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9e28756f78b0555a6733ffbf6ab98bd, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:58:05,921 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731751082172.d9e28756f78b0555a6733ffbf6ab98bd., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9e28756f78b0555a6733ffbf6ab98bd, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:58:05,921 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731751082172.d9e28756f78b0555a6733ffbf6ab98bd., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 because the exception is null or not the one we care about 2024-11-16T09:58:05,932 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/d9e28756f78b0555a6733ffbf6ab98bd/C/53be78a7cd2a4f1d98224b1c2affc854, entries=150, sequenceid=53, filesize=11.7 K 2024-11-16T09:58:05,937 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~67.09 KB/68700, heapSize ~176.48 KB/180720, currentSize=134.18 KB/137400 for d9e28756f78b0555a6733ffbf6ab98bd in 214ms, sequenceid=53, compaction requested=true 2024-11-16T09:58:05,937 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=37267 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=8 2024-11-16T09:58:05,938 DEBUG [RS_FLUSH_OPERATIONS-regionserver/037c58a19ddb:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731751082172.d9e28756f78b0555a6733ffbf6ab98bd. 2024-11-16T09:58:05,938 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for d9e28756f78b0555a6733ffbf6ab98bd: 2024-11-16T09:58:05,938 INFO [RS_FLUSH_OPERATIONS-regionserver/037c58a19ddb:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2902): Flushing d9e28756f78b0555a6733ffbf6ab98bd 3/3 column families, dataSize=134.18 KB heapSize=352.31 KB 2024-11-16T09:58:05,942 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store d9e28756f78b0555a6733ffbf6ab98bd:A, priority=-2147483648, current under compaction store size is 1 2024-11-16T09:58:05,944 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-16T09:58:05,944 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store d9e28756f78b0555a6733ffbf6ab98bd:B, priority=-2147483648, current under compaction store size is 2 2024-11-16T09:58:05,945 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-16T09:58:05,945 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store d9e28756f78b0555a6733ffbf6ab98bd:C, priority=-2147483648, current under compaction store size is 3 2024-11-16T09:58:05,945 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-16T09:58:05,947 DEBUG [RS:0;037c58a19ddb:37267-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-16T09:58:05,947 DEBUG [RS:0;037c58a19ddb:37267-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-16T09:58:05,953 DEBUG [RS:0;037c58a19ddb:37267-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36003 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-16T09:58:05,955 DEBUG [RS:0;037c58a19ddb:37267-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 38343 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-16T09:58:05,955 DEBUG [RS:0;037c58a19ddb:37267-shortCompactions-0 {}] regionserver.HStore(1541): d9e28756f78b0555a6733ffbf6ab98bd/A is initiating minor compaction (all files) 2024-11-16T09:58:05,955 DEBUG [RS:0;037c58a19ddb:37267-longCompactions-0 {}] regionserver.HStore(1541): d9e28756f78b0555a6733ffbf6ab98bd/B is initiating minor compaction (all files) 2024-11-16T09:58:05,956 INFO [RS:0;037c58a19ddb:37267-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of d9e28756f78b0555a6733ffbf6ab98bd/A in TestAcidGuarantees,,1731751082172.d9e28756f78b0555a6733ffbf6ab98bd. 2024-11-16T09:58:05,956 INFO [RS:0;037c58a19ddb:37267-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of d9e28756f78b0555a6733ffbf6ab98bd/B in TestAcidGuarantees,,1731751082172.d9e28756f78b0555a6733ffbf6ab98bd. 2024-11-16T09:58:05,957 INFO [RS:0;037c58a19ddb:37267-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/d9e28756f78b0555a6733ffbf6ab98bd/A/08ff50e864824ad0a3fd6cbd2d93846b, hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/d9e28756f78b0555a6733ffbf6ab98bd/A/263ffc1427114090a4723fb2b1539548, hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/d9e28756f78b0555a6733ffbf6ab98bd/A/fd8aea0564294c4d927efa1b14d23126] into tmpdir=hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/d9e28756f78b0555a6733ffbf6ab98bd/.tmp, totalSize=37.4 K 2024-11-16T09:58:05,957 INFO [RS:0;037c58a19ddb:37267-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/d9e28756f78b0555a6733ffbf6ab98bd/B/90cd50f304d648e2bb3f087e808ef5ce, hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/d9e28756f78b0555a6733ffbf6ab98bd/B/c21d5ad86bf84c7f9d79b1232a65de8b, hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/d9e28756f78b0555a6733ffbf6ab98bd/B/983f5ee278db4f3eaac85d7f97591fda] into tmpdir=hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/d9e28756f78b0555a6733ffbf6ab98bd/.tmp, totalSize=35.2 K 2024-11-16T09:58:05,959 DEBUG [RS:0;037c58a19ddb:37267-longCompactions-0 {}] compactions.Compactor(225): Compacting 90cd50f304d648e2bb3f087e808ef5ce, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=13, earliestPutTs=1731751084464 2024-11-16T09:58:05,960 DEBUG [RS:0;037c58a19ddb:37267-shortCompactions-0 {}] compactions.Compactor(225): Compacting 08ff50e864824ad0a3fd6cbd2d93846b, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=13, earliestPutTs=1731751084464 2024-11-16T09:58:05,960 DEBUG [RS:0;037c58a19ddb:37267-longCompactions-0 {}] compactions.Compactor(225): Compacting c21d5ad86bf84c7f9d79b1232a65de8b, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=40, earliestPutTs=1731751084655 2024-11-16T09:58:05,961 DEBUG [RS:0;037c58a19ddb:37267-shortCompactions-0 {}] compactions.Compactor(225): Compacting 263ffc1427114090a4723fb2b1539548, keycount=200, bloomtype=ROW, size=14.0 K, encoding=NONE, compression=NONE, seqNum=40, earliestPutTs=1731751084655 2024-11-16T09:58:05,961 DEBUG [RS:0;037c58a19ddb:37267-longCompactions-0 {}] compactions.Compactor(225): Compacting 983f5ee278db4f3eaac85d7f97591fda, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=53, earliestPutTs=1731751085061 2024-11-16T09:58:05,964 DEBUG [RS:0;037c58a19ddb:37267-shortCompactions-0 {}] compactions.Compactor(225): Compacting fd8aea0564294c4d927efa1b14d23126, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=53, earliestPutTs=1731751085061 2024-11-16T09:58:05,971 DEBUG [RS_FLUSH_OPERATIONS-regionserver/037c58a19ddb:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/d9e28756f78b0555a6733ffbf6ab98bd/.tmp/A/709c635778e04296bc2bbe6d259b56a0 is 50, key is test_row_0/A:col10/1731751085766/Put/seqid=0 2024-11-16T09:58:05,998 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37591 is added to blk_1073741846_1022 (size=12001) 2024-11-16T09:58:06,001 INFO [RS_FLUSH_OPERATIONS-regionserver/037c58a19ddb:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=76 (bloomFilter=true), to=hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/d9e28756f78b0555a6733ffbf6ab98bd/.tmp/A/709c635778e04296bc2bbe6d259b56a0 2024-11-16T09:58:06,029 DEBUG [RS_FLUSH_OPERATIONS-regionserver/037c58a19ddb:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/d9e28756f78b0555a6733ffbf6ab98bd/.tmp/B/15795d4f2c0e4fdc8b945a6f8860699b is 50, key is test_row_0/B:col10/1731751085766/Put/seqid=0 2024-11-16T09:58:06,053 INFO [RS:0;037c58a19ddb:37267-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): d9e28756f78b0555a6733ffbf6ab98bd#B#compaction#10 average throughput is 0.39 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-16T09:58:06,055 INFO [RS:0;037c58a19ddb:37267-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): d9e28756f78b0555a6733ffbf6ab98bd#A#compaction#11 average throughput is 0.50 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-16T09:58:06,055 DEBUG [RS:0;037c58a19ddb:37267-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/d9e28756f78b0555a6733ffbf6ab98bd/.tmp/A/8f9655c1866245c9a20f69a472c4ca2b is 50, key is test_row_0/A:col10/1731751085061/Put/seqid=0 2024-11-16T09:58:06,077 DEBUG [RS:0;037c58a19ddb:37267-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/d9e28756f78b0555a6733ffbf6ab98bd/.tmp/B/c986b11797e945d9b684c5e0c8adb5ef is 50, key is test_row_0/B:col10/1731751085061/Put/seqid=0 2024-11-16T09:58:06,077 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37591 is added to blk_1073741847_1023 (size=12001) 2024-11-16T09:58:06,096 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37591 is added to blk_1073741848_1024 (size=12104) 2024-11-16T09:58:06,117 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] regionserver.HRegion(8855): Flush requested on d9e28756f78b0555a6733ffbf6ab98bd 2024-11-16T09:58:06,117 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731751082172.d9e28756f78b0555a6733ffbf6ab98bd. as already flushing 2024-11-16T09:58:06,132 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37591 is added to blk_1073741849_1025 (size=12104) 2024-11-16T09:58:06,151 DEBUG [RS:0;037c58a19ddb:37267-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/d9e28756f78b0555a6733ffbf6ab98bd/.tmp/B/c986b11797e945d9b684c5e0c8adb5ef as hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/d9e28756f78b0555a6733ffbf6ab98bd/B/c986b11797e945d9b684c5e0c8adb5ef 2024-11-16T09:58:06,155 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9e28756f78b0555a6733ffbf6ab98bd, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T09:58:06,155 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39638 deadline: 1731751146143, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9e28756f78b0555a6733ffbf6ab98bd, server=037c58a19ddb,37267,1731751077844 2024-11-16T09:58:06,156 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731751082172.d9e28756f78b0555a6733ffbf6ab98bd., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 , the old value is region=TestAcidGuarantees,,1731751082172.d9e28756f78b0555a6733ffbf6ab98bd., hostname=037c58a19ddb,37267,1731751077844, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9e28756f78b0555a6733ffbf6ab98bd, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:58:06,157 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731751082172.d9e28756f78b0555a6733ffbf6ab98bd., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9e28756f78b0555a6733ffbf6ab98bd, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:58:06,157 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9e28756f78b0555a6733ffbf6ab98bd, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T09:58:06,157 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731751082172.d9e28756f78b0555a6733ffbf6ab98bd., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 because the exception is null or not the one we care about 2024-11-16T09:58:06,157 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39654 deadline: 1731751146147, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9e28756f78b0555a6733ffbf6ab98bd, server=037c58a19ddb,37267,1731751077844 2024-11-16T09:58:06,158 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731751082172.d9e28756f78b0555a6733ffbf6ab98bd., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 , the old value is region=TestAcidGuarantees,,1731751082172.d9e28756f78b0555a6733ffbf6ab98bd., hostname=037c58a19ddb,37267,1731751077844, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9e28756f78b0555a6733ffbf6ab98bd, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:58:06,158 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731751082172.d9e28756f78b0555a6733ffbf6ab98bd., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9e28756f78b0555a6733ffbf6ab98bd, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:58:06,158 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731751082172.d9e28756f78b0555a6733ffbf6ab98bd., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 because the exception is null or not the one we care about 2024-11-16T09:58:06,158 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9e28756f78b0555a6733ffbf6ab98bd, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T09:58:06,159 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39620 deadline: 1731751146151, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9e28756f78b0555a6733ffbf6ab98bd, server=037c58a19ddb,37267,1731751077844 2024-11-16T09:58:06,160 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731751082172.d9e28756f78b0555a6733ffbf6ab98bd., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 , the old value is region=TestAcidGuarantees,,1731751082172.d9e28756f78b0555a6733ffbf6ab98bd., hostname=037c58a19ddb,37267,1731751077844, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9e28756f78b0555a6733ffbf6ab98bd, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:58:06,160 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731751082172.d9e28756f78b0555a6733ffbf6ab98bd., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9e28756f78b0555a6733ffbf6ab98bd, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:58:06,160 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731751082172.d9e28756f78b0555a6733ffbf6ab98bd., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 because the exception is null or not the one we care about 2024-11-16T09:58:06,160 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9e28756f78b0555a6733ffbf6ab98bd, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T09:58:06,161 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39696 deadline: 1731751146154, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9e28756f78b0555a6733ffbf6ab98bd, server=037c58a19ddb,37267,1731751077844 2024-11-16T09:58:06,162 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731751082172.d9e28756f78b0555a6733ffbf6ab98bd., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 , the old value is region=TestAcidGuarantees,,1731751082172.d9e28756f78b0555a6733ffbf6ab98bd., hostname=037c58a19ddb,37267,1731751077844, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9e28756f78b0555a6733ffbf6ab98bd, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:58:06,162 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731751082172.d9e28756f78b0555a6733ffbf6ab98bd., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9e28756f78b0555a6733ffbf6ab98bd, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:58:06,162 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9e28756f78b0555a6733ffbf6ab98bd, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T09:58:06,162 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731751082172.d9e28756f78b0555a6733ffbf6ab98bd., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 because the exception is null or not the one we care about 2024-11-16T09:58:06,162 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39630 deadline: 1731751146155, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9e28756f78b0555a6733ffbf6ab98bd, server=037c58a19ddb,37267,1731751077844 2024-11-16T09:58:06,163 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731751082172.d9e28756f78b0555a6733ffbf6ab98bd., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 , the old value is region=TestAcidGuarantees,,1731751082172.d9e28756f78b0555a6733ffbf6ab98bd., hostname=037c58a19ddb,37267,1731751077844, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9e28756f78b0555a6733ffbf6ab98bd, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:58:06,163 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731751082172.d9e28756f78b0555a6733ffbf6ab98bd., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9e28756f78b0555a6733ffbf6ab98bd, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:58:06,163 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731751082172.d9e28756f78b0555a6733ffbf6ab98bd., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 because the exception is null or not the one we care about 2024-11-16T09:58:06,184 INFO [RS:0;037c58a19ddb:37267-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in d9e28756f78b0555a6733ffbf6ab98bd/B of d9e28756f78b0555a6733ffbf6ab98bd into c986b11797e945d9b684c5e0c8adb5ef(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-16T09:58:06,184 DEBUG [RS:0;037c58a19ddb:37267-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for d9e28756f78b0555a6733ffbf6ab98bd: 2024-11-16T09:58:06,189 INFO [RS:0;037c58a19ddb:37267-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731751082172.d9e28756f78b0555a6733ffbf6ab98bd., storeName=d9e28756f78b0555a6733ffbf6ab98bd/B, priority=13, startTime=1731751085944; duration=0sec 2024-11-16T09:58:06,190 DEBUG [RS:0;037c58a19ddb:37267-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-16T09:58:06,190 DEBUG [RS:0;037c58a19ddb:37267-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: d9e28756f78b0555a6733ffbf6ab98bd:B 2024-11-16T09:58:06,191 DEBUG [RS:0;037c58a19ddb:37267-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-16T09:58:06,195 DEBUG [RS:0;037c58a19ddb:37267-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36003 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-16T09:58:06,196 DEBUG [RS:0;037c58a19ddb:37267-longCompactions-0 {}] regionserver.HStore(1541): d9e28756f78b0555a6733ffbf6ab98bd/C is initiating minor compaction (all files) 2024-11-16T09:58:06,196 INFO [RS:0;037c58a19ddb:37267-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of d9e28756f78b0555a6733ffbf6ab98bd/C in TestAcidGuarantees,,1731751082172.d9e28756f78b0555a6733ffbf6ab98bd. 2024-11-16T09:58:06,196 INFO [RS:0;037c58a19ddb:37267-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/d9e28756f78b0555a6733ffbf6ab98bd/C/0eabc0bd610f4b8fb928684c5386127e, hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/d9e28756f78b0555a6733ffbf6ab98bd/C/1537924938324874ace183a7fc8c33fb, hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/d9e28756f78b0555a6733ffbf6ab98bd/C/53be78a7cd2a4f1d98224b1c2affc854] into tmpdir=hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/d9e28756f78b0555a6733ffbf6ab98bd/.tmp, totalSize=35.2 K 2024-11-16T09:58:06,197 DEBUG [RS:0;037c58a19ddb:37267-longCompactions-0 {}] compactions.Compactor(225): Compacting 0eabc0bd610f4b8fb928684c5386127e, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=13, earliestPutTs=1731751084464 2024-11-16T09:58:06,198 DEBUG [RS:0;037c58a19ddb:37267-longCompactions-0 {}] compactions.Compactor(225): Compacting 1537924938324874ace183a7fc8c33fb, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=40, earliestPutTs=1731751084655 2024-11-16T09:58:06,199 DEBUG [RS:0;037c58a19ddb:37267-longCompactions-0 {}] compactions.Compactor(225): Compacting 53be78a7cd2a4f1d98224b1c2affc854, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=53, earliestPutTs=1731751085061 2024-11-16T09:58:06,234 INFO [RS:0;037c58a19ddb:37267-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): d9e28756f78b0555a6733ffbf6ab98bd#C#compaction#13 average throughput is 1.31 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-16T09:58:06,236 DEBUG [RS:0;037c58a19ddb:37267-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/d9e28756f78b0555a6733ffbf6ab98bd/.tmp/C/4828c9b1f4464dccaaa081ce503fa018 is 50, key is test_row_0/C:col10/1731751085061/Put/seqid=0 2024-11-16T09:58:06,259 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37591 is added to blk_1073741850_1026 (size=12104) 2024-11-16T09:58:06,267 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9e28756f78b0555a6733ffbf6ab98bd, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T09:58:06,268 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39638 deadline: 1731751146264, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9e28756f78b0555a6733ffbf6ab98bd, server=037c58a19ddb,37267,1731751077844 2024-11-16T09:58:06,269 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731751082172.d9e28756f78b0555a6733ffbf6ab98bd., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 , the old value is region=TestAcidGuarantees,,1731751082172.d9e28756f78b0555a6733ffbf6ab98bd., hostname=037c58a19ddb,37267,1731751077844, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9e28756f78b0555a6733ffbf6ab98bd, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:58:06,269 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731751082172.d9e28756f78b0555a6733ffbf6ab98bd., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9e28756f78b0555a6733ffbf6ab98bd, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:58:06,269 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731751082172.d9e28756f78b0555a6733ffbf6ab98bd., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 because the exception is null or not the one we care about 2024-11-16T09:58:06,269 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9e28756f78b0555a6733ffbf6ab98bd, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T09:58:06,270 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39654 deadline: 1731751146264, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9e28756f78b0555a6733ffbf6ab98bd, server=037c58a19ddb,37267,1731751077844 2024-11-16T09:58:06,271 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9e28756f78b0555a6733ffbf6ab98bd, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T09:58:06,271 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39620 deadline: 1731751146265, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9e28756f78b0555a6733ffbf6ab98bd, server=037c58a19ddb,37267,1731751077844 2024-11-16T09:58:06,272 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731751082172.d9e28756f78b0555a6733ffbf6ab98bd., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 , the old value is region=TestAcidGuarantees,,1731751082172.d9e28756f78b0555a6733ffbf6ab98bd., hostname=037c58a19ddb,37267,1731751077844, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9e28756f78b0555a6733ffbf6ab98bd, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:58:06,272 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731751082172.d9e28756f78b0555a6733ffbf6ab98bd., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9e28756f78b0555a6733ffbf6ab98bd, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:58:06,272 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731751082172.d9e28756f78b0555a6733ffbf6ab98bd., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 because the exception is null or not the one we care about 2024-11-16T09:58:06,272 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731751082172.d9e28756f78b0555a6733ffbf6ab98bd., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 , the old value is region=TestAcidGuarantees,,1731751082172.d9e28756f78b0555a6733ffbf6ab98bd., hostname=037c58a19ddb,37267,1731751077844, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9e28756f78b0555a6733ffbf6ab98bd, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:58:06,273 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731751082172.d9e28756f78b0555a6733ffbf6ab98bd., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9e28756f78b0555a6733ffbf6ab98bd, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:58:06,273 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731751082172.d9e28756f78b0555a6733ffbf6ab98bd., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 because the exception is null or not the one we care about 2024-11-16T09:58:06,281 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9e28756f78b0555a6733ffbf6ab98bd, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T09:58:06,281 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39696 deadline: 1731751146274, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9e28756f78b0555a6733ffbf6ab98bd, server=037c58a19ddb,37267,1731751077844 2024-11-16T09:58:06,282 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731751082172.d9e28756f78b0555a6733ffbf6ab98bd., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 , the old value is region=TestAcidGuarantees,,1731751082172.d9e28756f78b0555a6733ffbf6ab98bd., hostname=037c58a19ddb,37267,1731751077844, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9e28756f78b0555a6733ffbf6ab98bd, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:58:06,282 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731751082172.d9e28756f78b0555a6733ffbf6ab98bd., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9e28756f78b0555a6733ffbf6ab98bd, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:58:06,282 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731751082172.d9e28756f78b0555a6733ffbf6ab98bd., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 because the exception is null or not the one we care about 2024-11-16T09:58:06,283 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9e28756f78b0555a6733ffbf6ab98bd, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T09:58:06,283 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39630 deadline: 1731751146274, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9e28756f78b0555a6733ffbf6ab98bd, server=037c58a19ddb,37267,1731751077844 2024-11-16T09:58:06,284 DEBUG [RS:0;037c58a19ddb:37267-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/d9e28756f78b0555a6733ffbf6ab98bd/.tmp/C/4828c9b1f4464dccaaa081ce503fa018 as hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/d9e28756f78b0555a6733ffbf6ab98bd/C/4828c9b1f4464dccaaa081ce503fa018 2024-11-16T09:58:06,284 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731751082172.d9e28756f78b0555a6733ffbf6ab98bd., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 , the old value is region=TestAcidGuarantees,,1731751082172.d9e28756f78b0555a6733ffbf6ab98bd., hostname=037c58a19ddb,37267,1731751077844, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9e28756f78b0555a6733ffbf6ab98bd, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:58:06,284 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731751082172.d9e28756f78b0555a6733ffbf6ab98bd., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9e28756f78b0555a6733ffbf6ab98bd, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:58:06,284 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731751082172.d9e28756f78b0555a6733ffbf6ab98bd., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 because the exception is null or not the one we care about 2024-11-16T09:58:06,301 INFO [RS:0;037c58a19ddb:37267-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in d9e28756f78b0555a6733ffbf6ab98bd/C of d9e28756f78b0555a6733ffbf6ab98bd into 4828c9b1f4464dccaaa081ce503fa018(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-16T09:58:06,302 DEBUG [RS:0;037c58a19ddb:37267-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for d9e28756f78b0555a6733ffbf6ab98bd: 2024-11-16T09:58:06,302 INFO [RS:0;037c58a19ddb:37267-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731751082172.d9e28756f78b0555a6733ffbf6ab98bd., storeName=d9e28756f78b0555a6733ffbf6ab98bd/C, priority=13, startTime=1731751085945; duration=0sec 2024-11-16T09:58:06,302 DEBUG [RS:0;037c58a19ddb:37267-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-16T09:58:06,302 DEBUG [RS:0;037c58a19ddb:37267-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: d9e28756f78b0555a6733ffbf6ab98bd:C 2024-11-16T09:58:06,477 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9e28756f78b0555a6733ffbf6ab98bd, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T09:58:06,477 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39638 deadline: 1731751146474, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9e28756f78b0555a6733ffbf6ab98bd, server=037c58a19ddb,37267,1731751077844 2024-11-16T09:58:06,480 INFO [RS_FLUSH_OPERATIONS-regionserver/037c58a19ddb:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=76 (bloomFilter=true), to=hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/d9e28756f78b0555a6733ffbf6ab98bd/.tmp/B/15795d4f2c0e4fdc8b945a6f8860699b 2024-11-16T09:58:06,481 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731751082172.d9e28756f78b0555a6733ffbf6ab98bd., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 , the old value is region=TestAcidGuarantees,,1731751082172.d9e28756f78b0555a6733ffbf6ab98bd., hostname=037c58a19ddb,37267,1731751077844, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9e28756f78b0555a6733ffbf6ab98bd, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:58:06,481 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731751082172.d9e28756f78b0555a6733ffbf6ab98bd., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9e28756f78b0555a6733ffbf6ab98bd, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:58:06,481 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731751082172.d9e28756f78b0555a6733ffbf6ab98bd., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 because the exception is null or not the one we care about 2024-11-16T09:58:06,489 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9e28756f78b0555a6733ffbf6ab98bd, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T09:58:06,489 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9e28756f78b0555a6733ffbf6ab98bd, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T09:58:06,489 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39654 deadline: 1731751146485, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9e28756f78b0555a6733ffbf6ab98bd, server=037c58a19ddb,37267,1731751077844 2024-11-16T09:58:06,489 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39620 deadline: 1731751146485, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9e28756f78b0555a6733ffbf6ab98bd, server=037c58a19ddb,37267,1731751077844 2024-11-16T09:58:06,490 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731751082172.d9e28756f78b0555a6733ffbf6ab98bd., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 , the old value is region=TestAcidGuarantees,,1731751082172.d9e28756f78b0555a6733ffbf6ab98bd., hostname=037c58a19ddb,37267,1731751077844, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9e28756f78b0555a6733ffbf6ab98bd, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:58:06,490 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731751082172.d9e28756f78b0555a6733ffbf6ab98bd., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9e28756f78b0555a6733ffbf6ab98bd, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:58:06,490 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731751082172.d9e28756f78b0555a6733ffbf6ab98bd., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 because the exception is null or not the one we care about 2024-11-16T09:58:06,490 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731751082172.d9e28756f78b0555a6733ffbf6ab98bd., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 , the old value is region=TestAcidGuarantees,,1731751082172.d9e28756f78b0555a6733ffbf6ab98bd., hostname=037c58a19ddb,37267,1731751077844, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9e28756f78b0555a6733ffbf6ab98bd, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:58:06,491 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731751082172.d9e28756f78b0555a6733ffbf6ab98bd., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9e28756f78b0555a6733ffbf6ab98bd, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:58:06,491 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731751082172.d9e28756f78b0555a6733ffbf6ab98bd., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 because the exception is null or not the one we care about 2024-11-16T09:58:06,499 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9e28756f78b0555a6733ffbf6ab98bd, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T09:58:06,500 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39696 deadline: 1731751146495, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9e28756f78b0555a6733ffbf6ab98bd, server=037c58a19ddb,37267,1731751077844 2024-11-16T09:58:06,500 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731751082172.d9e28756f78b0555a6733ffbf6ab98bd., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 , the old value is region=TestAcidGuarantees,,1731751082172.d9e28756f78b0555a6733ffbf6ab98bd., hostname=037c58a19ddb,37267,1731751077844, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9e28756f78b0555a6733ffbf6ab98bd, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:58:06,500 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731751082172.d9e28756f78b0555a6733ffbf6ab98bd., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9e28756f78b0555a6733ffbf6ab98bd, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:58:06,501 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731751082172.d9e28756f78b0555a6733ffbf6ab98bd., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 because the exception is null or not the one we care about 2024-11-16T09:58:06,501 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9e28756f78b0555a6733ffbf6ab98bd, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T09:58:06,502 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39630 deadline: 1731751146495, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9e28756f78b0555a6733ffbf6ab98bd, server=037c58a19ddb,37267,1731751077844 2024-11-16T09:58:06,502 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731751082172.d9e28756f78b0555a6733ffbf6ab98bd., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 , the old value is region=TestAcidGuarantees,,1731751082172.d9e28756f78b0555a6733ffbf6ab98bd., hostname=037c58a19ddb,37267,1731751077844, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9e28756f78b0555a6733ffbf6ab98bd, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:58:06,502 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731751082172.d9e28756f78b0555a6733ffbf6ab98bd., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9e28756f78b0555a6733ffbf6ab98bd, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:58:06,502 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731751082172.d9e28756f78b0555a6733ffbf6ab98bd., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 because the exception is null or not the one we care about 2024-11-16T09:58:06,519 DEBUG [RS:0;037c58a19ddb:37267-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/d9e28756f78b0555a6733ffbf6ab98bd/.tmp/A/8f9655c1866245c9a20f69a472c4ca2b as hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/d9e28756f78b0555a6733ffbf6ab98bd/A/8f9655c1866245c9a20f69a472c4ca2b 2024-11-16T09:58:06,522 DEBUG [RS_FLUSH_OPERATIONS-regionserver/037c58a19ddb:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/d9e28756f78b0555a6733ffbf6ab98bd/.tmp/C/7e5b4afd436c42ddaeae1ac5915abcc6 is 50, key is test_row_0/C:col10/1731751085766/Put/seqid=0 2024-11-16T09:58:06,536 INFO [RS:0;037c58a19ddb:37267-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in d9e28756f78b0555a6733ffbf6ab98bd/A of d9e28756f78b0555a6733ffbf6ab98bd into 8f9655c1866245c9a20f69a472c4ca2b(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-16T09:58:06,536 DEBUG [RS:0;037c58a19ddb:37267-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for d9e28756f78b0555a6733ffbf6ab98bd: 2024-11-16T09:58:06,536 INFO [RS:0;037c58a19ddb:37267-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731751082172.d9e28756f78b0555a6733ffbf6ab98bd., storeName=d9e28756f78b0555a6733ffbf6ab98bd/A, priority=13, startTime=1731751085941; duration=0sec 2024-11-16T09:58:06,536 DEBUG [RS:0;037c58a19ddb:37267-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-16T09:58:06,537 DEBUG [RS:0;037c58a19ddb:37267-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: d9e28756f78b0555a6733ffbf6ab98bd:A 2024-11-16T09:58:06,551 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37591 is added to blk_1073741851_1027 (size=12001) 2024-11-16T09:58:06,553 INFO [RS_FLUSH_OPERATIONS-regionserver/037c58a19ddb:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=76 (bloomFilter=true), to=hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/d9e28756f78b0555a6733ffbf6ab98bd/.tmp/C/7e5b4afd436c42ddaeae1ac5915abcc6 2024-11-16T09:58:06,565 DEBUG [RS_FLUSH_OPERATIONS-regionserver/037c58a19ddb:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/d9e28756f78b0555a6733ffbf6ab98bd/.tmp/A/709c635778e04296bc2bbe6d259b56a0 as hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/d9e28756f78b0555a6733ffbf6ab98bd/A/709c635778e04296bc2bbe6d259b56a0 2024-11-16T09:58:06,577 INFO [RS_FLUSH_OPERATIONS-regionserver/037c58a19ddb:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/d9e28756f78b0555a6733ffbf6ab98bd/A/709c635778e04296bc2bbe6d259b56a0, entries=150, sequenceid=76, filesize=11.7 K 2024-11-16T09:58:06,580 DEBUG [RS_FLUSH_OPERATIONS-regionserver/037c58a19ddb:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/d9e28756f78b0555a6733ffbf6ab98bd/.tmp/B/15795d4f2c0e4fdc8b945a6f8860699b as hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/d9e28756f78b0555a6733ffbf6ab98bd/B/15795d4f2c0e4fdc8b945a6f8860699b 2024-11-16T09:58:06,592 INFO [RS_FLUSH_OPERATIONS-regionserver/037c58a19ddb:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/d9e28756f78b0555a6733ffbf6ab98bd/B/15795d4f2c0e4fdc8b945a6f8860699b, entries=150, sequenceid=76, filesize=11.7 K 2024-11-16T09:58:06,594 DEBUG [RS_FLUSH_OPERATIONS-regionserver/037c58a19ddb:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/d9e28756f78b0555a6733ffbf6ab98bd/.tmp/C/7e5b4afd436c42ddaeae1ac5915abcc6 as hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/d9e28756f78b0555a6733ffbf6ab98bd/C/7e5b4afd436c42ddaeae1ac5915abcc6 2024-11-16T09:58:06,613 INFO [RS_FLUSH_OPERATIONS-regionserver/037c58a19ddb:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/d9e28756f78b0555a6733ffbf6ab98bd/C/7e5b4afd436c42ddaeae1ac5915abcc6, entries=150, sequenceid=76, filesize=11.7 K 2024-11-16T09:58:06,613 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37877 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=7 2024-11-16T09:58:06,616 INFO [RS_FLUSH_OPERATIONS-regionserver/037c58a19ddb:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(3140): Finished flush of dataSize ~134.18 KB/137400, heapSize ~352.27 KB/360720, currentSize=73.80 KB/75570 for d9e28756f78b0555a6733ffbf6ab98bd in 678ms, sequenceid=76, compaction requested=false 2024-11-16T09:58:06,616 DEBUG [RS_FLUSH_OPERATIONS-regionserver/037c58a19ddb:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2603): Flush status journal for d9e28756f78b0555a6733ffbf6ab98bd: 2024-11-16T09:58:06,616 DEBUG [RS_FLUSH_OPERATIONS-regionserver/037c58a19ddb:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731751082172.d9e28756f78b0555a6733ffbf6ab98bd. 2024-11-16T09:58:06,616 DEBUG [RS_FLUSH_OPERATIONS-regionserver/037c58a19ddb:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=8 2024-11-16T09:58:06,617 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37877 {}] master.HMaster(4169): Remote procedure done, pid=8 2024-11-16T09:58:06,618 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'hbase:meta' 2024-11-16T09:58:06,626 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=8, resume processing ppid=7 2024-11-16T09:58:06,626 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=8, ppid=7, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.1510 sec 2024-11-16T09:58:06,634 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=7, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=7, table=TestAcidGuarantees in 2.1800 sec 2024-11-16T09:58:06,806 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] regionserver.HRegion(8855): Flush requested on d9e28756f78b0555a6733ffbf6ab98bd 2024-11-16T09:58:06,808 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-16T09:58:06,808 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing d9e28756f78b0555a6733ffbf6ab98bd 3/3 column families, dataSize=87.22 KB heapSize=229.27 KB 2024-11-16T09:58:06,832 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/d9e28756f78b0555a6733ffbf6ab98bd/.tmp/A/6089e65fdbf4499499621e82097c69ab is 50, key is test_row_0/A:col10/1731751086797/Put/seqid=0 2024-11-16T09:58:06,870 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9e28756f78b0555a6733ffbf6ab98bd, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T09:58:06,872 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39638 deadline: 1731751146857, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9e28756f78b0555a6733ffbf6ab98bd, server=037c58a19ddb,37267,1731751077844 2024-11-16T09:58:06,871 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9e28756f78b0555a6733ffbf6ab98bd, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T09:58:06,872 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39696 deadline: 1731751146860, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9e28756f78b0555a6733ffbf6ab98bd, server=037c58a19ddb,37267,1731751077844 2024-11-16T09:58:06,872 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731751082172.d9e28756f78b0555a6733ffbf6ab98bd., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 , the old value is region=TestAcidGuarantees,,1731751082172.d9e28756f78b0555a6733ffbf6ab98bd., hostname=037c58a19ddb,37267,1731751077844, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9e28756f78b0555a6733ffbf6ab98bd, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:58:06,872 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731751082172.d9e28756f78b0555a6733ffbf6ab98bd., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9e28756f78b0555a6733ffbf6ab98bd, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:58:06,873 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731751082172.d9e28756f78b0555a6733ffbf6ab98bd., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 because the exception is null or not the one we care about 2024-11-16T09:58:06,873 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37591 is added to blk_1073741852_1028 (size=12001) 2024-11-16T09:58:06,873 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731751082172.d9e28756f78b0555a6733ffbf6ab98bd., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 , the old value is region=TestAcidGuarantees,,1731751082172.d9e28756f78b0555a6733ffbf6ab98bd., hostname=037c58a19ddb,37267,1731751077844, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9e28756f78b0555a6733ffbf6ab98bd, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:58:06,873 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731751082172.d9e28756f78b0555a6733ffbf6ab98bd., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9e28756f78b0555a6733ffbf6ab98bd, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:58:06,873 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731751082172.d9e28756f78b0555a6733ffbf6ab98bd., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 because the exception is null or not the one we care about 2024-11-16T09:58:06,873 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9e28756f78b0555a6733ffbf6ab98bd, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T09:58:06,873 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39654 deadline: 1731751146864, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9e28756f78b0555a6733ffbf6ab98bd, server=037c58a19ddb,37267,1731751077844 2024-11-16T09:58:06,874 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731751082172.d9e28756f78b0555a6733ffbf6ab98bd., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 , the old value is region=TestAcidGuarantees,,1731751082172.d9e28756f78b0555a6733ffbf6ab98bd., hostname=037c58a19ddb,37267,1731751077844, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9e28756f78b0555a6733ffbf6ab98bd, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:58:06,874 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9e28756f78b0555a6733ffbf6ab98bd, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T09:58:06,874 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=96 (bloomFilter=true), to=hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/d9e28756f78b0555a6733ffbf6ab98bd/.tmp/A/6089e65fdbf4499499621e82097c69ab 2024-11-16T09:58:06,874 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731751082172.d9e28756f78b0555a6733ffbf6ab98bd., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9e28756f78b0555a6733ffbf6ab98bd, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:58:06,874 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731751082172.d9e28756f78b0555a6733ffbf6ab98bd., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 because the exception is null or not the one we care about 2024-11-16T09:58:06,874 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9e28756f78b0555a6733ffbf6ab98bd, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T09:58:06,875 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39620 deadline: 1731751146869, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9e28756f78b0555a6733ffbf6ab98bd, server=037c58a19ddb,37267,1731751077844 2024-11-16T09:58:06,875 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39630 deadline: 1731751146861, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9e28756f78b0555a6733ffbf6ab98bd, server=037c58a19ddb,37267,1731751077844 2024-11-16T09:58:06,875 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731751082172.d9e28756f78b0555a6733ffbf6ab98bd., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 , the old value is region=TestAcidGuarantees,,1731751082172.d9e28756f78b0555a6733ffbf6ab98bd., hostname=037c58a19ddb,37267,1731751077844, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9e28756f78b0555a6733ffbf6ab98bd, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:58:06,875 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731751082172.d9e28756f78b0555a6733ffbf6ab98bd., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9e28756f78b0555a6733ffbf6ab98bd, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:58:06,876 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731751082172.d9e28756f78b0555a6733ffbf6ab98bd., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 because the exception is null or not the one we care about 2024-11-16T09:58:06,876 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731751082172.d9e28756f78b0555a6733ffbf6ab98bd., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 , the old value is region=TestAcidGuarantees,,1731751082172.d9e28756f78b0555a6733ffbf6ab98bd., hostname=037c58a19ddb,37267,1731751077844, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9e28756f78b0555a6733ffbf6ab98bd, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:58:06,876 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731751082172.d9e28756f78b0555a6733ffbf6ab98bd., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9e28756f78b0555a6733ffbf6ab98bd, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:58:06,876 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731751082172.d9e28756f78b0555a6733ffbf6ab98bd., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 because the exception is null or not the one we care about 2024-11-16T09:58:06,894 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/d9e28756f78b0555a6733ffbf6ab98bd/.tmp/B/52c98c4ef2174fe3879c8bf25ea45439 is 50, key is test_row_0/B:col10/1731751086797/Put/seqid=0 2024-11-16T09:58:06,912 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37591 is added to blk_1073741853_1029 (size=12001) 2024-11-16T09:58:06,916 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=96 (bloomFilter=true), to=hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/d9e28756f78b0555a6733ffbf6ab98bd/.tmp/B/52c98c4ef2174fe3879c8bf25ea45439 2024-11-16T09:58:06,942 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/d9e28756f78b0555a6733ffbf6ab98bd/.tmp/C/8ffd381310ac430bb3fa37b8738b0544 is 50, key is test_row_0/C:col10/1731751086797/Put/seqid=0 2024-11-16T09:58:06,965 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37591 is added to blk_1073741854_1030 (size=12001) 2024-11-16T09:58:06,969 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=96 (bloomFilter=true), to=hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/d9e28756f78b0555a6733ffbf6ab98bd/.tmp/C/8ffd381310ac430bb3fa37b8738b0544 2024-11-16T09:58:06,985 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9e28756f78b0555a6733ffbf6ab98bd, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T09:58:06,985 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39654 deadline: 1731751146983, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9e28756f78b0555a6733ffbf6ab98bd, server=037c58a19ddb,37267,1731751077844 2024-11-16T09:58:06,986 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9e28756f78b0555a6733ffbf6ab98bd, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T09:58:06,986 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731751082172.d9e28756f78b0555a6733ffbf6ab98bd., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 , the old value is region=TestAcidGuarantees,,1731751082172.d9e28756f78b0555a6733ffbf6ab98bd., hostname=037c58a19ddb,37267,1731751077844, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9e28756f78b0555a6733ffbf6ab98bd, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:58:06,986 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731751082172.d9e28756f78b0555a6733ffbf6ab98bd., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9e28756f78b0555a6733ffbf6ab98bd, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:58:06,986 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731751082172.d9e28756f78b0555a6733ffbf6ab98bd., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 because the exception is null or not the one we care about 2024-11-16T09:58:06,986 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39620 deadline: 1731751146984, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9e28756f78b0555a6733ffbf6ab98bd, server=037c58a19ddb,37267,1731751077844 2024-11-16T09:58:06,987 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/d9e28756f78b0555a6733ffbf6ab98bd/.tmp/A/6089e65fdbf4499499621e82097c69ab as hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/d9e28756f78b0555a6733ffbf6ab98bd/A/6089e65fdbf4499499621e82097c69ab 2024-11-16T09:58:06,987 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731751082172.d9e28756f78b0555a6733ffbf6ab98bd., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 , the old value is region=TestAcidGuarantees,,1731751082172.d9e28756f78b0555a6733ffbf6ab98bd., hostname=037c58a19ddb,37267,1731751077844, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9e28756f78b0555a6733ffbf6ab98bd, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:58:06,987 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731751082172.d9e28756f78b0555a6733ffbf6ab98bd., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9e28756f78b0555a6733ffbf6ab98bd, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:58:06,987 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731751082172.d9e28756f78b0555a6733ffbf6ab98bd., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 because the exception is null or not the one we care about 2024-11-16T09:58:06,987 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9e28756f78b0555a6733ffbf6ab98bd, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T09:58:06,987 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39696 deadline: 1731751146984, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9e28756f78b0555a6733ffbf6ab98bd, server=037c58a19ddb,37267,1731751077844 2024-11-16T09:58:06,988 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731751082172.d9e28756f78b0555a6733ffbf6ab98bd., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 , the old value is region=TestAcidGuarantees,,1731751082172.d9e28756f78b0555a6733ffbf6ab98bd., hostname=037c58a19ddb,37267,1731751077844, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9e28756f78b0555a6733ffbf6ab98bd, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:58:06,988 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9e28756f78b0555a6733ffbf6ab98bd, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T09:58:06,988 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9e28756f78b0555a6733ffbf6ab98bd, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T09:58:06,988 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731751082172.d9e28756f78b0555a6733ffbf6ab98bd., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9e28756f78b0555a6733ffbf6ab98bd, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:58:06,988 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731751082172.d9e28756f78b0555a6733ffbf6ab98bd., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 because the exception is null or not the one we care about 2024-11-16T09:58:06,988 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39630 deadline: 1731751146985, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9e28756f78b0555a6733ffbf6ab98bd, server=037c58a19ddb,37267,1731751077844 2024-11-16T09:58:06,988 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39638 deadline: 1731751146984, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9e28756f78b0555a6733ffbf6ab98bd, server=037c58a19ddb,37267,1731751077844 2024-11-16T09:58:06,989 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731751082172.d9e28756f78b0555a6733ffbf6ab98bd., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 , the old value is region=TestAcidGuarantees,,1731751082172.d9e28756f78b0555a6733ffbf6ab98bd., hostname=037c58a19ddb,37267,1731751077844, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9e28756f78b0555a6733ffbf6ab98bd, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:58:06,989 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731751082172.d9e28756f78b0555a6733ffbf6ab98bd., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9e28756f78b0555a6733ffbf6ab98bd, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:58:06,989 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731751082172.d9e28756f78b0555a6733ffbf6ab98bd., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 because the exception is null or not the one we care about 2024-11-16T09:58:06,989 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731751082172.d9e28756f78b0555a6733ffbf6ab98bd., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 , the old value is region=TestAcidGuarantees,,1731751082172.d9e28756f78b0555a6733ffbf6ab98bd., hostname=037c58a19ddb,37267,1731751077844, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9e28756f78b0555a6733ffbf6ab98bd, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:58:06,989 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731751082172.d9e28756f78b0555a6733ffbf6ab98bd., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9e28756f78b0555a6733ffbf6ab98bd, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:58:06,989 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731751082172.d9e28756f78b0555a6733ffbf6ab98bd., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 because the exception is null or not the one we care about 2024-11-16T09:58:07,004 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/d9e28756f78b0555a6733ffbf6ab98bd/A/6089e65fdbf4499499621e82097c69ab, entries=150, sequenceid=96, filesize=11.7 K 2024-11-16T09:58:07,006 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/d9e28756f78b0555a6733ffbf6ab98bd/.tmp/B/52c98c4ef2174fe3879c8bf25ea45439 as hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/d9e28756f78b0555a6733ffbf6ab98bd/B/52c98c4ef2174fe3879c8bf25ea45439 2024-11-16T09:58:07,021 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/d9e28756f78b0555a6733ffbf6ab98bd/B/52c98c4ef2174fe3879c8bf25ea45439, entries=150, sequenceid=96, filesize=11.7 K 2024-11-16T09:58:07,025 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/d9e28756f78b0555a6733ffbf6ab98bd/.tmp/C/8ffd381310ac430bb3fa37b8738b0544 as hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/d9e28756f78b0555a6733ffbf6ab98bd/C/8ffd381310ac430bb3fa37b8738b0544 2024-11-16T09:58:07,038 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/d9e28756f78b0555a6733ffbf6ab98bd/C/8ffd381310ac430bb3fa37b8738b0544, entries=150, sequenceid=96, filesize=11.7 K 2024-11-16T09:58:07,040 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~93.93 KB/96180, heapSize ~246.80 KB/252720, currentSize=107.34 KB/109920 for d9e28756f78b0555a6733ffbf6ab98bd in 232ms, sequenceid=96, compaction requested=true 2024-11-16T09:58:07,041 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for d9e28756f78b0555a6733ffbf6ab98bd: 2024-11-16T09:58:07,041 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store d9e28756f78b0555a6733ffbf6ab98bd:A, priority=-2147483648, current under compaction store size is 1 2024-11-16T09:58:07,041 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-16T09:58:07,041 DEBUG [RS:0;037c58a19ddb:37267-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-16T09:58:07,041 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store d9e28756f78b0555a6733ffbf6ab98bd:B, priority=-2147483648, current under compaction store size is 2 2024-11-16T09:58:07,041 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-16T09:58:07,041 DEBUG [RS:0;037c58a19ddb:37267-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-16T09:58:07,041 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store d9e28756f78b0555a6733ffbf6ab98bd:C, priority=-2147483648, current under compaction store size is 3 2024-11-16T09:58:07,042 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-16T09:58:07,044 DEBUG [RS:0;037c58a19ddb:37267-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36106 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-16T09:58:07,044 DEBUG [RS:0;037c58a19ddb:37267-longCompactions-0 {}] regionserver.HStore(1541): d9e28756f78b0555a6733ffbf6ab98bd/B is initiating minor compaction (all files) 2024-11-16T09:58:07,044 INFO [RS:0;037c58a19ddb:37267-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of d9e28756f78b0555a6733ffbf6ab98bd/B in TestAcidGuarantees,,1731751082172.d9e28756f78b0555a6733ffbf6ab98bd. 2024-11-16T09:58:07,045 INFO [RS:0;037c58a19ddb:37267-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/d9e28756f78b0555a6733ffbf6ab98bd/B/c986b11797e945d9b684c5e0c8adb5ef, hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/d9e28756f78b0555a6733ffbf6ab98bd/B/15795d4f2c0e4fdc8b945a6f8860699b, hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/d9e28756f78b0555a6733ffbf6ab98bd/B/52c98c4ef2174fe3879c8bf25ea45439] into tmpdir=hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/d9e28756f78b0555a6733ffbf6ab98bd/.tmp, totalSize=35.3 K 2024-11-16T09:58:07,045 DEBUG [RS:0;037c58a19ddb:37267-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36106 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-16T09:58:07,045 DEBUG [RS:0;037c58a19ddb:37267-shortCompactions-0 {}] regionserver.HStore(1541): d9e28756f78b0555a6733ffbf6ab98bd/A is initiating minor compaction (all files) 2024-11-16T09:58:07,045 INFO [RS:0;037c58a19ddb:37267-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of d9e28756f78b0555a6733ffbf6ab98bd/A in TestAcidGuarantees,,1731751082172.d9e28756f78b0555a6733ffbf6ab98bd. 2024-11-16T09:58:07,046 INFO [RS:0;037c58a19ddb:37267-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/d9e28756f78b0555a6733ffbf6ab98bd/A/8f9655c1866245c9a20f69a472c4ca2b, hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/d9e28756f78b0555a6733ffbf6ab98bd/A/709c635778e04296bc2bbe6d259b56a0, hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/d9e28756f78b0555a6733ffbf6ab98bd/A/6089e65fdbf4499499621e82097c69ab] into tmpdir=hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/d9e28756f78b0555a6733ffbf6ab98bd/.tmp, totalSize=35.3 K 2024-11-16T09:58:07,046 DEBUG [RS:0;037c58a19ddb:37267-longCompactions-0 {}] compactions.Compactor(225): Compacting c986b11797e945d9b684c5e0c8adb5ef, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=53, earliestPutTs=1731751085061 2024-11-16T09:58:07,047 DEBUG [RS:0;037c58a19ddb:37267-shortCompactions-0 {}] compactions.Compactor(225): Compacting 8f9655c1866245c9a20f69a472c4ca2b, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=53, earliestPutTs=1731751085061 2024-11-16T09:58:07,048 DEBUG [RS:0;037c58a19ddb:37267-longCompactions-0 {}] compactions.Compactor(225): Compacting 15795d4f2c0e4fdc8b945a6f8860699b, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=76, earliestPutTs=1731751085766 2024-11-16T09:58:07,048 DEBUG [RS:0;037c58a19ddb:37267-shortCompactions-0 {}] compactions.Compactor(225): Compacting 709c635778e04296bc2bbe6d259b56a0, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=76, earliestPutTs=1731751085766 2024-11-16T09:58:07,049 DEBUG [RS:0;037c58a19ddb:37267-longCompactions-0 {}] compactions.Compactor(225): Compacting 52c98c4ef2174fe3879c8bf25ea45439, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=96, earliestPutTs=1731751086150 2024-11-16T09:58:07,049 DEBUG [RS:0;037c58a19ddb:37267-shortCompactions-0 {}] compactions.Compactor(225): Compacting 6089e65fdbf4499499621e82097c69ab, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=96, earliestPutTs=1731751086150 2024-11-16T09:58:07,077 INFO [RS:0;037c58a19ddb:37267-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): d9e28756f78b0555a6733ffbf6ab98bd#A#compaction#19 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-16T09:58:07,078 DEBUG [RS:0;037c58a19ddb:37267-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/d9e28756f78b0555a6733ffbf6ab98bd/.tmp/A/5bdf9040556249c8890d8df9eff5b42c is 50, key is test_row_0/A:col10/1731751086797/Put/seqid=0 2024-11-16T09:58:07,078 INFO [RS:0;037c58a19ddb:37267-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): d9e28756f78b0555a6733ffbf6ab98bd#B#compaction#18 average throughput is 1.31 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-16T09:58:07,079 DEBUG [RS:0;037c58a19ddb:37267-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/d9e28756f78b0555a6733ffbf6ab98bd/.tmp/B/284e61e32b8d466384dd3636d4d4fbb6 is 50, key is test_row_0/B:col10/1731751086797/Put/seqid=0 2024-11-16T09:58:07,098 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37591 is added to blk_1073741856_1032 (size=12207) 2024-11-16T09:58:07,106 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37591 is added to blk_1073741855_1031 (size=12207) 2024-11-16T09:58:07,115 DEBUG [RS:0;037c58a19ddb:37267-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/d9e28756f78b0555a6733ffbf6ab98bd/.tmp/A/5bdf9040556249c8890d8df9eff5b42c as hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/d9e28756f78b0555a6733ffbf6ab98bd/A/5bdf9040556249c8890d8df9eff5b42c 2024-11-16T09:58:07,122 DEBUG [RS:0;037c58a19ddb:37267-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/d9e28756f78b0555a6733ffbf6ab98bd/.tmp/B/284e61e32b8d466384dd3636d4d4fbb6 as hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/d9e28756f78b0555a6733ffbf6ab98bd/B/284e61e32b8d466384dd3636d4d4fbb6 2024-11-16T09:58:07,131 INFO [RS:0;037c58a19ddb:37267-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in d9e28756f78b0555a6733ffbf6ab98bd/A of d9e28756f78b0555a6733ffbf6ab98bd into 5bdf9040556249c8890d8df9eff5b42c(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-16T09:58:07,133 DEBUG [RS:0;037c58a19ddb:37267-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for d9e28756f78b0555a6733ffbf6ab98bd: 2024-11-16T09:58:07,133 INFO [RS:0;037c58a19ddb:37267-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731751082172.d9e28756f78b0555a6733ffbf6ab98bd., storeName=d9e28756f78b0555a6733ffbf6ab98bd/A, priority=13, startTime=1731751087041; duration=0sec 2024-11-16T09:58:07,133 DEBUG [RS:0;037c58a19ddb:37267-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-16T09:58:07,134 DEBUG [RS:0;037c58a19ddb:37267-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: d9e28756f78b0555a6733ffbf6ab98bd:A 2024-11-16T09:58:07,134 DEBUG [RS:0;037c58a19ddb:37267-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-16T09:58:07,137 INFO [RS:0;037c58a19ddb:37267-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in d9e28756f78b0555a6733ffbf6ab98bd/B of d9e28756f78b0555a6733ffbf6ab98bd into 284e61e32b8d466384dd3636d4d4fbb6(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-16T09:58:07,137 DEBUG [RS:0;037c58a19ddb:37267-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for d9e28756f78b0555a6733ffbf6ab98bd: 2024-11-16T09:58:07,137 INFO [RS:0;037c58a19ddb:37267-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731751082172.d9e28756f78b0555a6733ffbf6ab98bd., storeName=d9e28756f78b0555a6733ffbf6ab98bd/B, priority=13, startTime=1731751087041; duration=0sec 2024-11-16T09:58:07,137 DEBUG [RS:0;037c58a19ddb:37267-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-16T09:58:07,137 DEBUG [RS:0;037c58a19ddb:37267-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: d9e28756f78b0555a6733ffbf6ab98bd:B 2024-11-16T09:58:07,139 DEBUG [RS:0;037c58a19ddb:37267-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36106 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-16T09:58:07,139 DEBUG [RS:0;037c58a19ddb:37267-shortCompactions-0 {}] regionserver.HStore(1541): d9e28756f78b0555a6733ffbf6ab98bd/C is initiating minor compaction (all files) 2024-11-16T09:58:07,139 INFO [RS:0;037c58a19ddb:37267-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of d9e28756f78b0555a6733ffbf6ab98bd/C in TestAcidGuarantees,,1731751082172.d9e28756f78b0555a6733ffbf6ab98bd. 2024-11-16T09:58:07,140 INFO [RS:0;037c58a19ddb:37267-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/d9e28756f78b0555a6733ffbf6ab98bd/C/4828c9b1f4464dccaaa081ce503fa018, hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/d9e28756f78b0555a6733ffbf6ab98bd/C/7e5b4afd436c42ddaeae1ac5915abcc6, hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/d9e28756f78b0555a6733ffbf6ab98bd/C/8ffd381310ac430bb3fa37b8738b0544] into tmpdir=hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/d9e28756f78b0555a6733ffbf6ab98bd/.tmp, totalSize=35.3 K 2024-11-16T09:58:07,142 DEBUG [RS:0;037c58a19ddb:37267-shortCompactions-0 {}] compactions.Compactor(225): Compacting 4828c9b1f4464dccaaa081ce503fa018, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=53, earliestPutTs=1731751085061 2024-11-16T09:58:07,142 DEBUG [RS:0;037c58a19ddb:37267-shortCompactions-0 {}] compactions.Compactor(225): Compacting 7e5b4afd436c42ddaeae1ac5915abcc6, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=76, earliestPutTs=1731751085766 2024-11-16T09:58:07,143 DEBUG [RS:0;037c58a19ddb:37267-shortCompactions-0 {}] compactions.Compactor(225): Compacting 8ffd381310ac430bb3fa37b8738b0544, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=96, earliestPutTs=1731751086150 2024-11-16T09:58:07,160 INFO [RS:0;037c58a19ddb:37267-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): d9e28756f78b0555a6733ffbf6ab98bd#C#compaction#20 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-16T09:58:07,161 DEBUG [RS:0;037c58a19ddb:37267-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/d9e28756f78b0555a6733ffbf6ab98bd/.tmp/C/aba8c9e7079440bebfb145d1604cd540 is 50, key is test_row_0/C:col10/1731751086797/Put/seqid=0 2024-11-16T09:58:07,180 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37591 is added to blk_1073741857_1033 (size=12207) 2024-11-16T09:58:07,197 DEBUG [RS:0;037c58a19ddb:37267-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/d9e28756f78b0555a6733ffbf6ab98bd/.tmp/C/aba8c9e7079440bebfb145d1604cd540 as hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/d9e28756f78b0555a6733ffbf6ab98bd/C/aba8c9e7079440bebfb145d1604cd540 2024-11-16T09:58:07,199 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-16T09:58:07,199 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing d9e28756f78b0555a6733ffbf6ab98bd 3/3 column families, dataSize=120.76 KB heapSize=317.16 KB 2024-11-16T09:58:07,201 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] regionserver.HRegion(8855): Flush requested on d9e28756f78b0555a6733ffbf6ab98bd 2024-11-16T09:58:07,215 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/d9e28756f78b0555a6733ffbf6ab98bd/.tmp/A/ef206033b0624c7a94ea8929d5fca761 is 50, key is test_row_0/A:col10/1731751087195/Put/seqid=0 2024-11-16T09:58:07,224 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37591 is added to blk_1073741858_1034 (size=12001) 2024-11-16T09:58:07,257 INFO [RS:0;037c58a19ddb:37267-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in d9e28756f78b0555a6733ffbf6ab98bd/C of d9e28756f78b0555a6733ffbf6ab98bd into aba8c9e7079440bebfb145d1604cd540(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-16T09:58:07,257 DEBUG [RS:0;037c58a19ddb:37267-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for d9e28756f78b0555a6733ffbf6ab98bd: 2024-11-16T09:58:07,257 INFO [RS:0;037c58a19ddb:37267-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731751082172.d9e28756f78b0555a6733ffbf6ab98bd., storeName=d9e28756f78b0555a6733ffbf6ab98bd/C, priority=13, startTime=1731751087041; duration=0sec 2024-11-16T09:58:07,257 DEBUG [RS:0;037c58a19ddb:37267-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-16T09:58:07,257 DEBUG [RS:0;037c58a19ddb:37267-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: d9e28756f78b0555a6733ffbf6ab98bd:C 2024-11-16T09:58:07,272 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-11-16T09:58:07,286 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9e28756f78b0555a6733ffbf6ab98bd, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T09:58:07,287 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39630 deadline: 1731751147278, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9e28756f78b0555a6733ffbf6ab98bd, server=037c58a19ddb,37267,1731751077844 2024-11-16T09:58:07,288 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731751082172.d9e28756f78b0555a6733ffbf6ab98bd., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 , the old value is region=TestAcidGuarantees,,1731751082172.d9e28756f78b0555a6733ffbf6ab98bd., hostname=037c58a19ddb,37267,1731751077844, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9e28756f78b0555a6733ffbf6ab98bd, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:58:07,288 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731751082172.d9e28756f78b0555a6733ffbf6ab98bd., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9e28756f78b0555a6733ffbf6ab98bd, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:58:07,288 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731751082172.d9e28756f78b0555a6733ffbf6ab98bd., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 because the exception is null or not the one we care about 2024-11-16T09:58:07,289 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9e28756f78b0555a6733ffbf6ab98bd, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T09:58:07,289 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39620 deadline: 1731751147278, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9e28756f78b0555a6733ffbf6ab98bd, server=037c58a19ddb,37267,1731751077844 2024-11-16T09:58:07,290 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731751082172.d9e28756f78b0555a6733ffbf6ab98bd., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 , the old value is region=TestAcidGuarantees,,1731751082172.d9e28756f78b0555a6733ffbf6ab98bd., hostname=037c58a19ddb,37267,1731751077844, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9e28756f78b0555a6733ffbf6ab98bd, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:58:07,290 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731751082172.d9e28756f78b0555a6733ffbf6ab98bd., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9e28756f78b0555a6733ffbf6ab98bd, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:58:07,290 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731751082172.d9e28756f78b0555a6733ffbf6ab98bd., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 because the exception is null or not the one we care about 2024-11-16T09:58:07,290 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9e28756f78b0555a6733ffbf6ab98bd, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T09:58:07,290 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39654 deadline: 1731751147278, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9e28756f78b0555a6733ffbf6ab98bd, server=037c58a19ddb,37267,1731751077844 2024-11-16T09:58:07,291 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731751082172.d9e28756f78b0555a6733ffbf6ab98bd., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 , the old value is region=TestAcidGuarantees,,1731751082172.d9e28756f78b0555a6733ffbf6ab98bd., hostname=037c58a19ddb,37267,1731751077844, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9e28756f78b0555a6733ffbf6ab98bd, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:58:07,291 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731751082172.d9e28756f78b0555a6733ffbf6ab98bd., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9e28756f78b0555a6733ffbf6ab98bd, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:58:07,291 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731751082172.d9e28756f78b0555a6733ffbf6ab98bd., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 because the exception is null or not the one we care about 2024-11-16T09:58:07,292 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9e28756f78b0555a6733ffbf6ab98bd, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T09:58:07,293 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39696 deadline: 1731751147283, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9e28756f78b0555a6733ffbf6ab98bd, server=037c58a19ddb,37267,1731751077844 2024-11-16T09:58:07,293 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731751082172.d9e28756f78b0555a6733ffbf6ab98bd., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 , the old value is region=TestAcidGuarantees,,1731751082172.d9e28756f78b0555a6733ffbf6ab98bd., hostname=037c58a19ddb,37267,1731751077844, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9e28756f78b0555a6733ffbf6ab98bd, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:58:07,293 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731751082172.d9e28756f78b0555a6733ffbf6ab98bd., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9e28756f78b0555a6733ffbf6ab98bd, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:58:07,293 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731751082172.d9e28756f78b0555a6733ffbf6ab98bd., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 because the exception is null or not the one we care about 2024-11-16T09:58:07,294 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9e28756f78b0555a6733ffbf6ab98bd, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T09:58:07,296 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39638 deadline: 1731751147287, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9e28756f78b0555a6733ffbf6ab98bd, server=037c58a19ddb,37267,1731751077844 2024-11-16T09:58:07,297 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731751082172.d9e28756f78b0555a6733ffbf6ab98bd., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 , the old value is region=TestAcidGuarantees,,1731751082172.d9e28756f78b0555a6733ffbf6ab98bd., hostname=037c58a19ddb,37267,1731751077844, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9e28756f78b0555a6733ffbf6ab98bd, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:58:07,297 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731751082172.d9e28756f78b0555a6733ffbf6ab98bd., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9e28756f78b0555a6733ffbf6ab98bd, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:58:07,297 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731751082172.d9e28756f78b0555a6733ffbf6ab98bd., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 because the exception is null or not the one we care about 2024-11-16T09:58:07,397 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9e28756f78b0555a6733ffbf6ab98bd, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T09:58:07,398 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9e28756f78b0555a6733ffbf6ab98bd, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T09:58:07,400 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39630 deadline: 1731751147393, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9e28756f78b0555a6733ffbf6ab98bd, server=037c58a19ddb,37267,1731751077844 2024-11-16T09:58:07,400 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39620 deadline: 1731751147394, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9e28756f78b0555a6733ffbf6ab98bd, server=037c58a19ddb,37267,1731751077844 2024-11-16T09:58:07,401 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731751082172.d9e28756f78b0555a6733ffbf6ab98bd., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 , the old value is region=TestAcidGuarantees,,1731751082172.d9e28756f78b0555a6733ffbf6ab98bd., hostname=037c58a19ddb,37267,1731751077844, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9e28756f78b0555a6733ffbf6ab98bd, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:58:07,401 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731751082172.d9e28756f78b0555a6733ffbf6ab98bd., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9e28756f78b0555a6733ffbf6ab98bd, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:58:07,401 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731751082172.d9e28756f78b0555a6733ffbf6ab98bd., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 because the exception is null or not the one we care about 2024-11-16T09:58:07,401 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731751082172.d9e28756f78b0555a6733ffbf6ab98bd., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 , the old value is region=TestAcidGuarantees,,1731751082172.d9e28756f78b0555a6733ffbf6ab98bd., hostname=037c58a19ddb,37267,1731751077844, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9e28756f78b0555a6733ffbf6ab98bd, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:58:07,401 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731751082172.d9e28756f78b0555a6733ffbf6ab98bd., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9e28756f78b0555a6733ffbf6ab98bd, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:58:07,401 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731751082172.d9e28756f78b0555a6733ffbf6ab98bd., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 because the exception is null or not the one we care about 2024-11-16T09:58:07,404 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9e28756f78b0555a6733ffbf6ab98bd, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T09:58:07,405 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39654 deadline: 1731751147403, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9e28756f78b0555a6733ffbf6ab98bd, server=037c58a19ddb,37267,1731751077844 2024-11-16T09:58:07,406 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731751082172.d9e28756f78b0555a6733ffbf6ab98bd., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 , the old value is region=TestAcidGuarantees,,1731751082172.d9e28756f78b0555a6733ffbf6ab98bd., hostname=037c58a19ddb,37267,1731751077844, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9e28756f78b0555a6733ffbf6ab98bd, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:58:07,405 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9e28756f78b0555a6733ffbf6ab98bd, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T09:58:07,406 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731751082172.d9e28756f78b0555a6733ffbf6ab98bd., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9e28756f78b0555a6733ffbf6ab98bd, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:58:07,406 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731751082172.d9e28756f78b0555a6733ffbf6ab98bd., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 because the exception is null or not the one we care about 2024-11-16T09:58:07,406 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39696 deadline: 1731751147403, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9e28756f78b0555a6733ffbf6ab98bd, server=037c58a19ddb,37267,1731751077844 2024-11-16T09:58:07,406 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731751082172.d9e28756f78b0555a6733ffbf6ab98bd., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 , the old value is region=TestAcidGuarantees,,1731751082172.d9e28756f78b0555a6733ffbf6ab98bd., hostname=037c58a19ddb,37267,1731751077844, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9e28756f78b0555a6733ffbf6ab98bd, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:58:07,406 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9e28756f78b0555a6733ffbf6ab98bd, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T09:58:07,406 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731751082172.d9e28756f78b0555a6733ffbf6ab98bd., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9e28756f78b0555a6733ffbf6ab98bd, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:58:07,407 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731751082172.d9e28756f78b0555a6733ffbf6ab98bd., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 because the exception is null or not the one we care about 2024-11-16T09:58:07,407 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39638 deadline: 1731751147404, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9e28756f78b0555a6733ffbf6ab98bd, server=037c58a19ddb,37267,1731751077844 2024-11-16T09:58:07,407 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731751082172.d9e28756f78b0555a6733ffbf6ab98bd., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 , the old value is region=TestAcidGuarantees,,1731751082172.d9e28756f78b0555a6733ffbf6ab98bd., hostname=037c58a19ddb,37267,1731751077844, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9e28756f78b0555a6733ffbf6ab98bd, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:58:07,407 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731751082172.d9e28756f78b0555a6733ffbf6ab98bd., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9e28756f78b0555a6733ffbf6ab98bd, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:58:07,407 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731751082172.d9e28756f78b0555a6733ffbf6ab98bd., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 because the exception is null or not the one we care about 2024-11-16T09:58:07,539 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_hbase_table_meta 2024-11-16T09:58:07,540 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_hbase_table_meta Metrics about Tables on a single HBase RegionServer 2024-11-16T09:58:07,543 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=Coprocessor.Region.CP_org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint 2024-11-16T09:58:07,543 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=Coprocessor.Region.CP_org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint Metrics about HBase RegionObservers 2024-11-16T09:58:07,543 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_master_table_store 2024-11-16T09:58:07,543 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_master_table_store Metrics about Tables on a single HBase RegionServer 2024-11-16T09:58:07,548 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_TestAcidGuarantees 2024-11-16T09:58:07,548 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_default_table_TestAcidGuarantees Metrics about Tables on a single HBase RegionServer 2024-11-16T09:58:07,605 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9e28756f78b0555a6733ffbf6ab98bd, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T09:58:07,605 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39630 deadline: 1731751147603, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9e28756f78b0555a6733ffbf6ab98bd, server=037c58a19ddb,37267,1731751077844 2024-11-16T09:58:07,606 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731751082172.d9e28756f78b0555a6733ffbf6ab98bd., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 , the old value is region=TestAcidGuarantees,,1731751082172.d9e28756f78b0555a6733ffbf6ab98bd., hostname=037c58a19ddb,37267,1731751077844, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9e28756f78b0555a6733ffbf6ab98bd, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:58:07,606 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731751082172.d9e28756f78b0555a6733ffbf6ab98bd., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9e28756f78b0555a6733ffbf6ab98bd, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:58:07,606 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731751082172.d9e28756f78b0555a6733ffbf6ab98bd., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 because the exception is null or not the one we care about 2024-11-16T09:58:07,614 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9e28756f78b0555a6733ffbf6ab98bd, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T09:58:07,615 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39620 deadline: 1731751147613, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9e28756f78b0555a6733ffbf6ab98bd, server=037c58a19ddb,37267,1731751077844 2024-11-16T09:58:07,615 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9e28756f78b0555a6733ffbf6ab98bd, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T09:58:07,615 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39696 deadline: 1731751147614, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9e28756f78b0555a6733ffbf6ab98bd, server=037c58a19ddb,37267,1731751077844 2024-11-16T09:58:07,615 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731751082172.d9e28756f78b0555a6733ffbf6ab98bd., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 , the old value is region=TestAcidGuarantees,,1731751082172.d9e28756f78b0555a6733ffbf6ab98bd., hostname=037c58a19ddb,37267,1731751077844, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9e28756f78b0555a6733ffbf6ab98bd, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:58:07,615 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731751082172.d9e28756f78b0555a6733ffbf6ab98bd., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9e28756f78b0555a6733ffbf6ab98bd, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:58:07,615 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731751082172.d9e28756f78b0555a6733ffbf6ab98bd., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 because the exception is null or not the one we care about 2024-11-16T09:58:07,616 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9e28756f78b0555a6733ffbf6ab98bd, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T09:58:07,616 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39638 deadline: 1731751147614, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9e28756f78b0555a6733ffbf6ab98bd, server=037c58a19ddb,37267,1731751077844 2024-11-16T09:58:07,616 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731751082172.d9e28756f78b0555a6733ffbf6ab98bd., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 , the old value is region=TestAcidGuarantees,,1731751082172.d9e28756f78b0555a6733ffbf6ab98bd., hostname=037c58a19ddb,37267,1731751077844, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9e28756f78b0555a6733ffbf6ab98bd, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:58:07,616 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731751082172.d9e28756f78b0555a6733ffbf6ab98bd., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9e28756f78b0555a6733ffbf6ab98bd, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:58:07,616 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731751082172.d9e28756f78b0555a6733ffbf6ab98bd., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 because the exception is null or not the one we care about 2024-11-16T09:58:07,616 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9e28756f78b0555a6733ffbf6ab98bd, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T09:58:07,617 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731751082172.d9e28756f78b0555a6733ffbf6ab98bd., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 , the old value is region=TestAcidGuarantees,,1731751082172.d9e28756f78b0555a6733ffbf6ab98bd., hostname=037c58a19ddb,37267,1731751077844, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9e28756f78b0555a6733ffbf6ab98bd, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:58:07,617 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39654 deadline: 1731751147615, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9e28756f78b0555a6733ffbf6ab98bd, server=037c58a19ddb,37267,1731751077844 2024-11-16T09:58:07,617 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731751082172.d9e28756f78b0555a6733ffbf6ab98bd., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9e28756f78b0555a6733ffbf6ab98bd, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:58:07,617 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731751082172.d9e28756f78b0555a6733ffbf6ab98bd., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 because the exception is null or not the one we care about 2024-11-16T09:58:07,617 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731751082172.d9e28756f78b0555a6733ffbf6ab98bd., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 , the old value is region=TestAcidGuarantees,,1731751082172.d9e28756f78b0555a6733ffbf6ab98bd., hostname=037c58a19ddb,37267,1731751077844, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9e28756f78b0555a6733ffbf6ab98bd, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:58:07,617 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731751082172.d9e28756f78b0555a6733ffbf6ab98bd., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9e28756f78b0555a6733ffbf6ab98bd, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:58:07,617 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731751082172.d9e28756f78b0555a6733ffbf6ab98bd., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 because the exception is null or not the one we care about 2024-11-16T09:58:07,628 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=119 (bloomFilter=true), to=hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/d9e28756f78b0555a6733ffbf6ab98bd/.tmp/A/ef206033b0624c7a94ea8929d5fca761 2024-11-16T09:58:07,668 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/d9e28756f78b0555a6733ffbf6ab98bd/.tmp/B/4d7c8b1b9e844592bd35ef66f2d8167b is 50, key is test_row_0/B:col10/1731751087195/Put/seqid=0 2024-11-16T09:58:07,686 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37591 is added to blk_1073741859_1035 (size=12001) 2024-11-16T09:58:07,688 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=119 (bloomFilter=true), to=hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/d9e28756f78b0555a6733ffbf6ab98bd/.tmp/B/4d7c8b1b9e844592bd35ef66f2d8167b 2024-11-16T09:58:07,707 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/d9e28756f78b0555a6733ffbf6ab98bd/.tmp/C/775aa7ebdf894f119f74fcbc7d28ea62 is 50, key is test_row_0/C:col10/1731751087195/Put/seqid=0 2024-11-16T09:58:07,726 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37591 is added to blk_1073741860_1036 (size=12001) 2024-11-16T09:58:07,727 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=119 (bloomFilter=true), to=hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/d9e28756f78b0555a6733ffbf6ab98bd/.tmp/C/775aa7ebdf894f119f74fcbc7d28ea62 2024-11-16T09:58:07,741 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/d9e28756f78b0555a6733ffbf6ab98bd/.tmp/A/ef206033b0624c7a94ea8929d5fca761 as hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/d9e28756f78b0555a6733ffbf6ab98bd/A/ef206033b0624c7a94ea8929d5fca761 2024-11-16T09:58:07,757 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/d9e28756f78b0555a6733ffbf6ab98bd/A/ef206033b0624c7a94ea8929d5fca761, entries=150, sequenceid=119, filesize=11.7 K 2024-11-16T09:58:07,759 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/d9e28756f78b0555a6733ffbf6ab98bd/.tmp/B/4d7c8b1b9e844592bd35ef66f2d8167b as hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/d9e28756f78b0555a6733ffbf6ab98bd/B/4d7c8b1b9e844592bd35ef66f2d8167b 2024-11-16T09:58:07,770 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/d9e28756f78b0555a6733ffbf6ab98bd/B/4d7c8b1b9e844592bd35ef66f2d8167b, entries=150, sequenceid=119, filesize=11.7 K 2024-11-16T09:58:07,784 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/d9e28756f78b0555a6733ffbf6ab98bd/.tmp/C/775aa7ebdf894f119f74fcbc7d28ea62 as hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/d9e28756f78b0555a6733ffbf6ab98bd/C/775aa7ebdf894f119f74fcbc7d28ea62 2024-11-16T09:58:07,799 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/d9e28756f78b0555a6733ffbf6ab98bd/C/775aa7ebdf894f119f74fcbc7d28ea62, entries=150, sequenceid=119, filesize=11.7 K 2024-11-16T09:58:07,801 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~120.76 KB/123660, heapSize ~317.11 KB/324720, currentSize=80.51 KB/82440 for d9e28756f78b0555a6733ffbf6ab98bd in 601ms, sequenceid=119, compaction requested=false 2024-11-16T09:58:07,801 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for d9e28756f78b0555a6733ffbf6ab98bd: 2024-11-16T09:58:07,926 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-16T09:58:07,926 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing d9e28756f78b0555a6733ffbf6ab98bd 3/3 column families, dataSize=93.93 KB heapSize=246.84 KB 2024-11-16T09:58:07,927 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] regionserver.HRegion(8855): Flush requested on d9e28756f78b0555a6733ffbf6ab98bd 2024-11-16T09:58:07,942 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/d9e28756f78b0555a6733ffbf6ab98bd/.tmp/A/43a771f3ee484bb4936eea002ad0ec6c is 50, key is test_row_0/A:col10/1731751087270/Put/seqid=0 2024-11-16T09:58:07,951 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37591 is added to blk_1073741861_1037 (size=16931) 2024-11-16T09:58:08,045 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9e28756f78b0555a6733ffbf6ab98bd, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T09:58:08,046 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39654 deadline: 1731751148037, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9e28756f78b0555a6733ffbf6ab98bd, server=037c58a19ddb,37267,1731751077844 2024-11-16T09:58:08,046 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9e28756f78b0555a6733ffbf6ab98bd, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T09:58:08,046 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39630 deadline: 1731751148037, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9e28756f78b0555a6733ffbf6ab98bd, server=037c58a19ddb,37267,1731751077844 2024-11-16T09:58:08,046 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9e28756f78b0555a6733ffbf6ab98bd, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T09:58:08,046 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731751082172.d9e28756f78b0555a6733ffbf6ab98bd., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 , the old value is region=TestAcidGuarantees,,1731751082172.d9e28756f78b0555a6733ffbf6ab98bd., hostname=037c58a19ddb,37267,1731751077844, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9e28756f78b0555a6733ffbf6ab98bd, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:58:08,046 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731751082172.d9e28756f78b0555a6733ffbf6ab98bd., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9e28756f78b0555a6733ffbf6ab98bd, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:58:08,047 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39620 deadline: 1731751148039, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9e28756f78b0555a6733ffbf6ab98bd, server=037c58a19ddb,37267,1731751077844 2024-11-16T09:58:08,047 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731751082172.d9e28756f78b0555a6733ffbf6ab98bd., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 because the exception is null or not the one we care about 2024-11-16T09:58:08,047 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731751082172.d9e28756f78b0555a6733ffbf6ab98bd., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 , the old value is region=TestAcidGuarantees,,1731751082172.d9e28756f78b0555a6733ffbf6ab98bd., hostname=037c58a19ddb,37267,1731751077844, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9e28756f78b0555a6733ffbf6ab98bd, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:58:08,047 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731751082172.d9e28756f78b0555a6733ffbf6ab98bd., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 , the old value is region=TestAcidGuarantees,,1731751082172.d9e28756f78b0555a6733ffbf6ab98bd., hostname=037c58a19ddb,37267,1731751077844, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9e28756f78b0555a6733ffbf6ab98bd, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:58:08,047 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731751082172.d9e28756f78b0555a6733ffbf6ab98bd., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9e28756f78b0555a6733ffbf6ab98bd, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:58:08,047 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731751082172.d9e28756f78b0555a6733ffbf6ab98bd., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9e28756f78b0555a6733ffbf6ab98bd, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:58:08,047 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731751082172.d9e28756f78b0555a6733ffbf6ab98bd., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 because the exception is null or not the one we care about 2024-11-16T09:58:08,047 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731751082172.d9e28756f78b0555a6733ffbf6ab98bd., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 because the exception is null or not the one we care about 2024-11-16T09:58:08,048 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9e28756f78b0555a6733ffbf6ab98bd, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T09:58:08,048 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39638 deadline: 1731751148045, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9e28756f78b0555a6733ffbf6ab98bd, server=037c58a19ddb,37267,1731751077844 2024-11-16T09:58:08,049 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731751082172.d9e28756f78b0555a6733ffbf6ab98bd., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 , the old value is region=TestAcidGuarantees,,1731751082172.d9e28756f78b0555a6733ffbf6ab98bd., hostname=037c58a19ddb,37267,1731751077844, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9e28756f78b0555a6733ffbf6ab98bd, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:58:08,049 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9e28756f78b0555a6733ffbf6ab98bd, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T09:58:08,049 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731751082172.d9e28756f78b0555a6733ffbf6ab98bd., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9e28756f78b0555a6733ffbf6ab98bd, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:58:08,049 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731751082172.d9e28756f78b0555a6733ffbf6ab98bd., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 because the exception is null or not the one we care about 2024-11-16T09:58:08,049 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39696 deadline: 1731751148045, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9e28756f78b0555a6733ffbf6ab98bd, server=037c58a19ddb,37267,1731751077844 2024-11-16T09:58:08,049 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731751082172.d9e28756f78b0555a6733ffbf6ab98bd., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 , the old value is region=TestAcidGuarantees,,1731751082172.d9e28756f78b0555a6733ffbf6ab98bd., hostname=037c58a19ddb,37267,1731751077844, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9e28756f78b0555a6733ffbf6ab98bd, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:58:08,050 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731751082172.d9e28756f78b0555a6733ffbf6ab98bd., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9e28756f78b0555a6733ffbf6ab98bd, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:58:08,050 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731751082172.d9e28756f78b0555a6733ffbf6ab98bd., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 because the exception is null or not the one we care about 2024-11-16T09:58:08,157 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9e28756f78b0555a6733ffbf6ab98bd, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T09:58:08,158 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39654 deadline: 1731751148153, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9e28756f78b0555a6733ffbf6ab98bd, server=037c58a19ddb,37267,1731751077844 2024-11-16T09:58:08,158 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731751082172.d9e28756f78b0555a6733ffbf6ab98bd., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 , the old value is region=TestAcidGuarantees,,1731751082172.d9e28756f78b0555a6733ffbf6ab98bd., hostname=037c58a19ddb,37267,1731751077844, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9e28756f78b0555a6733ffbf6ab98bd, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:58:08,158 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9e28756f78b0555a6733ffbf6ab98bd, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T09:58:08,158 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731751082172.d9e28756f78b0555a6733ffbf6ab98bd., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9e28756f78b0555a6733ffbf6ab98bd, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:58:08,158 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731751082172.d9e28756f78b0555a6733ffbf6ab98bd., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 because the exception is null or not the one we care about 2024-11-16T09:58:08,158 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39620 deadline: 1731751148153, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9e28756f78b0555a6733ffbf6ab98bd, server=037c58a19ddb,37267,1731751077844 2024-11-16T09:58:08,159 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9e28756f78b0555a6733ffbf6ab98bd, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T09:58:08,159 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39630 deadline: 1731751148154, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9e28756f78b0555a6733ffbf6ab98bd, server=037c58a19ddb,37267,1731751077844 2024-11-16T09:58:08,159 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731751082172.d9e28756f78b0555a6733ffbf6ab98bd., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 , the old value is region=TestAcidGuarantees,,1731751082172.d9e28756f78b0555a6733ffbf6ab98bd., hostname=037c58a19ddb,37267,1731751077844, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9e28756f78b0555a6733ffbf6ab98bd, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:58:08,159 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9e28756f78b0555a6733ffbf6ab98bd, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T09:58:08,159 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731751082172.d9e28756f78b0555a6733ffbf6ab98bd., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9e28756f78b0555a6733ffbf6ab98bd, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:58:08,159 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731751082172.d9e28756f78b0555a6733ffbf6ab98bd., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 because the exception is null or not the one we care about 2024-11-16T09:58:08,159 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39638 deadline: 1731751148154, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9e28756f78b0555a6733ffbf6ab98bd, server=037c58a19ddb,37267,1731751077844 2024-11-16T09:58:08,159 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9e28756f78b0555a6733ffbf6ab98bd, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T09:58:08,159 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731751082172.d9e28756f78b0555a6733ffbf6ab98bd., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 , the old value is region=TestAcidGuarantees,,1731751082172.d9e28756f78b0555a6733ffbf6ab98bd., hostname=037c58a19ddb,37267,1731751077844, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9e28756f78b0555a6733ffbf6ab98bd, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:58:08,159 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731751082172.d9e28756f78b0555a6733ffbf6ab98bd., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9e28756f78b0555a6733ffbf6ab98bd, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:58:08,160 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731751082172.d9e28756f78b0555a6733ffbf6ab98bd., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 because the exception is null or not the one we care about 2024-11-16T09:58:08,160 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39696 deadline: 1731751148154, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9e28756f78b0555a6733ffbf6ab98bd, server=037c58a19ddb,37267,1731751077844 2024-11-16T09:58:08,160 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731751082172.d9e28756f78b0555a6733ffbf6ab98bd., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 , the old value is region=TestAcidGuarantees,,1731751082172.d9e28756f78b0555a6733ffbf6ab98bd., hostname=037c58a19ddb,37267,1731751077844, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9e28756f78b0555a6733ffbf6ab98bd, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:58:08,160 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731751082172.d9e28756f78b0555a6733ffbf6ab98bd., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9e28756f78b0555a6733ffbf6ab98bd, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:58:08,160 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731751082172.d9e28756f78b0555a6733ffbf6ab98bd., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 because the exception is null or not the one we care about 2024-11-16T09:58:08,160 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731751082172.d9e28756f78b0555a6733ffbf6ab98bd., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 , the old value is region=TestAcidGuarantees,,1731751082172.d9e28756f78b0555a6733ffbf6ab98bd., hostname=037c58a19ddb,37267,1731751077844, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9e28756f78b0555a6733ffbf6ab98bd, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:58:08,160 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731751082172.d9e28756f78b0555a6733ffbf6ab98bd., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9e28756f78b0555a6733ffbf6ab98bd, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:58:08,160 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731751082172.d9e28756f78b0555a6733ffbf6ab98bd., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 because the exception is null or not the one we care about 2024-11-16T09:58:08,352 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=137 (bloomFilter=true), to=hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/d9e28756f78b0555a6733ffbf6ab98bd/.tmp/A/43a771f3ee484bb4936eea002ad0ec6c 2024-11-16T09:58:08,366 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9e28756f78b0555a6733ffbf6ab98bd, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T09:58:08,367 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39654 deadline: 1731751148363, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9e28756f78b0555a6733ffbf6ab98bd, server=037c58a19ddb,37267,1731751077844 2024-11-16T09:58:08,367 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731751082172.d9e28756f78b0555a6733ffbf6ab98bd., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 , the old value is region=TestAcidGuarantees,,1731751082172.d9e28756f78b0555a6733ffbf6ab98bd., hostname=037c58a19ddb,37267,1731751077844, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9e28756f78b0555a6733ffbf6ab98bd, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:58:08,367 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731751082172.d9e28756f78b0555a6733ffbf6ab98bd., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9e28756f78b0555a6733ffbf6ab98bd, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:58:08,367 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731751082172.d9e28756f78b0555a6733ffbf6ab98bd., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 because the exception is null or not the one we care about 2024-11-16T09:58:08,367 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9e28756f78b0555a6733ffbf6ab98bd, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T09:58:08,368 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39620 deadline: 1731751148364, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9e28756f78b0555a6733ffbf6ab98bd, server=037c58a19ddb,37267,1731751077844 2024-11-16T09:58:08,368 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731751082172.d9e28756f78b0555a6733ffbf6ab98bd., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 , the old value is region=TestAcidGuarantees,,1731751082172.d9e28756f78b0555a6733ffbf6ab98bd., hostname=037c58a19ddb,37267,1731751077844, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9e28756f78b0555a6733ffbf6ab98bd, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:58:08,368 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9e28756f78b0555a6733ffbf6ab98bd, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T09:58:08,368 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731751082172.d9e28756f78b0555a6733ffbf6ab98bd., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9e28756f78b0555a6733ffbf6ab98bd, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:58:08,368 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731751082172.d9e28756f78b0555a6733ffbf6ab98bd., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 because the exception is null or not the one we care about 2024-11-16T09:58:08,368 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39630 deadline: 1731751148364, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9e28756f78b0555a6733ffbf6ab98bd, server=037c58a19ddb,37267,1731751077844 2024-11-16T09:58:08,369 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731751082172.d9e28756f78b0555a6733ffbf6ab98bd., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 , the old value is region=TestAcidGuarantees,,1731751082172.d9e28756f78b0555a6733ffbf6ab98bd., hostname=037c58a19ddb,37267,1731751077844, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9e28756f78b0555a6733ffbf6ab98bd, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:58:08,369 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9e28756f78b0555a6733ffbf6ab98bd, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T09:58:08,369 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731751082172.d9e28756f78b0555a6733ffbf6ab98bd., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9e28756f78b0555a6733ffbf6ab98bd, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:58:08,369 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731751082172.d9e28756f78b0555a6733ffbf6ab98bd., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 because the exception is null or not the one we care about 2024-11-16T09:58:08,369 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39696 deadline: 1731751148364, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9e28756f78b0555a6733ffbf6ab98bd, server=037c58a19ddb,37267,1731751077844 2024-11-16T09:58:08,369 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731751082172.d9e28756f78b0555a6733ffbf6ab98bd., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 , the old value is region=TestAcidGuarantees,,1731751082172.d9e28756f78b0555a6733ffbf6ab98bd., hostname=037c58a19ddb,37267,1731751077844, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9e28756f78b0555a6733ffbf6ab98bd, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:58:08,369 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731751082172.d9e28756f78b0555a6733ffbf6ab98bd., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9e28756f78b0555a6733ffbf6ab98bd, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:58:08,370 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731751082172.d9e28756f78b0555a6733ffbf6ab98bd., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 because the exception is null or not the one we care about 2024-11-16T09:58:08,373 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/d9e28756f78b0555a6733ffbf6ab98bd/.tmp/B/e3e6fb7a08bc4bbda4fbb1276b27aba1 is 50, key is test_row_0/B:col10/1731751087270/Put/seqid=0 2024-11-16T09:58:08,375 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9e28756f78b0555a6733ffbf6ab98bd, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T09:58:08,375 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39638 deadline: 1731751148373, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9e28756f78b0555a6733ffbf6ab98bd, server=037c58a19ddb,37267,1731751077844 2024-11-16T09:58:08,375 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731751082172.d9e28756f78b0555a6733ffbf6ab98bd., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 , the old value is region=TestAcidGuarantees,,1731751082172.d9e28756f78b0555a6733ffbf6ab98bd., hostname=037c58a19ddb,37267,1731751077844, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9e28756f78b0555a6733ffbf6ab98bd, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:58:08,376 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731751082172.d9e28756f78b0555a6733ffbf6ab98bd., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9e28756f78b0555a6733ffbf6ab98bd, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:58:08,376 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731751082172.d9e28756f78b0555a6733ffbf6ab98bd., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 because the exception is null or not the one we care about 2024-11-16T09:58:08,386 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37591 is added to blk_1073741862_1038 (size=12151) 2024-11-16T09:58:08,387 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=137 (bloomFilter=true), to=hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/d9e28756f78b0555a6733ffbf6ab98bd/.tmp/B/e3e6fb7a08bc4bbda4fbb1276b27aba1 2024-11-16T09:58:08,410 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/d9e28756f78b0555a6733ffbf6ab98bd/.tmp/C/43f98f5a3e8c418080fd6b081c369f71 is 50, key is test_row_0/C:col10/1731751087270/Put/seqid=0 2024-11-16T09:58:08,430 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37591 is added to blk_1073741863_1039 (size=12151) 2024-11-16T09:58:08,433 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=137 (bloomFilter=true), to=hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/d9e28756f78b0555a6733ffbf6ab98bd/.tmp/C/43f98f5a3e8c418080fd6b081c369f71 2024-11-16T09:58:08,443 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/d9e28756f78b0555a6733ffbf6ab98bd/.tmp/A/43a771f3ee484bb4936eea002ad0ec6c as hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/d9e28756f78b0555a6733ffbf6ab98bd/A/43a771f3ee484bb4936eea002ad0ec6c 2024-11-16T09:58:08,455 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/d9e28756f78b0555a6733ffbf6ab98bd/A/43a771f3ee484bb4936eea002ad0ec6c, entries=250, sequenceid=137, filesize=16.5 K 2024-11-16T09:58:08,457 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/d9e28756f78b0555a6733ffbf6ab98bd/.tmp/B/e3e6fb7a08bc4bbda4fbb1276b27aba1 as hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/d9e28756f78b0555a6733ffbf6ab98bd/B/e3e6fb7a08bc4bbda4fbb1276b27aba1 2024-11-16T09:58:08,464 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/d9e28756f78b0555a6733ffbf6ab98bd/B/e3e6fb7a08bc4bbda4fbb1276b27aba1, entries=150, sequenceid=137, filesize=11.9 K 2024-11-16T09:58:08,467 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/d9e28756f78b0555a6733ffbf6ab98bd/.tmp/C/43f98f5a3e8c418080fd6b081c369f71 as hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/d9e28756f78b0555a6733ffbf6ab98bd/C/43f98f5a3e8c418080fd6b081c369f71 2024-11-16T09:58:08,477 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/d9e28756f78b0555a6733ffbf6ab98bd/C/43f98f5a3e8c418080fd6b081c369f71, entries=150, sequenceid=137, filesize=11.9 K 2024-11-16T09:58:08,480 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~93.93 KB/96180, heapSize ~246.80 KB/252720, currentSize=114.05 KB/116790 for d9e28756f78b0555a6733ffbf6ab98bd in 554ms, sequenceid=137, compaction requested=true 2024-11-16T09:58:08,480 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for d9e28756f78b0555a6733ffbf6ab98bd: 2024-11-16T09:58:08,480 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store d9e28756f78b0555a6733ffbf6ab98bd:A, priority=-2147483648, current under compaction store size is 1 2024-11-16T09:58:08,480 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-16T09:58:08,480 DEBUG [RS:0;037c58a19ddb:37267-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-16T09:58:08,480 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store d9e28756f78b0555a6733ffbf6ab98bd:B, priority=-2147483648, current under compaction store size is 2 2024-11-16T09:58:08,480 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-16T09:58:08,480 DEBUG [RS:0;037c58a19ddb:37267-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-16T09:58:08,481 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store d9e28756f78b0555a6733ffbf6ab98bd:C, priority=-2147483648, current under compaction store size is 3 2024-11-16T09:58:08,482 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-16T09:58:08,482 DEBUG [RS:0;037c58a19ddb:37267-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 41139 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-16T09:58:08,482 DEBUG [RS:0;037c58a19ddb:37267-shortCompactions-0 {}] regionserver.HStore(1541): d9e28756f78b0555a6733ffbf6ab98bd/A is initiating minor compaction (all files) 2024-11-16T09:58:08,483 INFO [RS:0;037c58a19ddb:37267-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of d9e28756f78b0555a6733ffbf6ab98bd/A in TestAcidGuarantees,,1731751082172.d9e28756f78b0555a6733ffbf6ab98bd. 2024-11-16T09:58:08,483 INFO [RS:0;037c58a19ddb:37267-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/d9e28756f78b0555a6733ffbf6ab98bd/A/5bdf9040556249c8890d8df9eff5b42c, hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/d9e28756f78b0555a6733ffbf6ab98bd/A/ef206033b0624c7a94ea8929d5fca761, hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/d9e28756f78b0555a6733ffbf6ab98bd/A/43a771f3ee484bb4936eea002ad0ec6c] into tmpdir=hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/d9e28756f78b0555a6733ffbf6ab98bd/.tmp, totalSize=40.2 K 2024-11-16T09:58:08,484 DEBUG [RS:0;037c58a19ddb:37267-shortCompactions-0 {}] compactions.Compactor(225): Compacting 5bdf9040556249c8890d8df9eff5b42c, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=96, earliestPutTs=1731751086150 2024-11-16T09:58:08,485 DEBUG [RS:0;037c58a19ddb:37267-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36359 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-16T09:58:08,485 DEBUG [RS:0;037c58a19ddb:37267-shortCompactions-0 {}] compactions.Compactor(225): Compacting ef206033b0624c7a94ea8929d5fca761, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=119, earliestPutTs=1731751086855 2024-11-16T09:58:08,485 DEBUG [RS:0;037c58a19ddb:37267-longCompactions-0 {}] regionserver.HStore(1541): d9e28756f78b0555a6733ffbf6ab98bd/B is initiating minor compaction (all files) 2024-11-16T09:58:08,485 INFO [RS:0;037c58a19ddb:37267-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of d9e28756f78b0555a6733ffbf6ab98bd/B in TestAcidGuarantees,,1731751082172.d9e28756f78b0555a6733ffbf6ab98bd. 2024-11-16T09:58:08,486 INFO [RS:0;037c58a19ddb:37267-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/d9e28756f78b0555a6733ffbf6ab98bd/B/284e61e32b8d466384dd3636d4d4fbb6, hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/d9e28756f78b0555a6733ffbf6ab98bd/B/4d7c8b1b9e844592bd35ef66f2d8167b, hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/d9e28756f78b0555a6733ffbf6ab98bd/B/e3e6fb7a08bc4bbda4fbb1276b27aba1] into tmpdir=hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/d9e28756f78b0555a6733ffbf6ab98bd/.tmp, totalSize=35.5 K 2024-11-16T09:58:08,486 DEBUG [RS:0;037c58a19ddb:37267-shortCompactions-0 {}] compactions.Compactor(225): Compacting 43a771f3ee484bb4936eea002ad0ec6c, keycount=250, bloomtype=ROW, size=16.5 K, encoding=NONE, compression=NONE, seqNum=137, earliestPutTs=1731751087270 2024-11-16T09:58:08,487 DEBUG [RS:0;037c58a19ddb:37267-longCompactions-0 {}] compactions.Compactor(225): Compacting 284e61e32b8d466384dd3636d4d4fbb6, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=96, earliestPutTs=1731751086150 2024-11-16T09:58:08,488 DEBUG [RS:0;037c58a19ddb:37267-longCompactions-0 {}] compactions.Compactor(225): Compacting 4d7c8b1b9e844592bd35ef66f2d8167b, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=119, earliestPutTs=1731751086855 2024-11-16T09:58:08,489 DEBUG [RS:0;037c58a19ddb:37267-longCompactions-0 {}] compactions.Compactor(225): Compacting e3e6fb7a08bc4bbda4fbb1276b27aba1, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=137, earliestPutTs=1731751087270 2024-11-16T09:58:08,508 INFO [RS:0;037c58a19ddb:37267-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): d9e28756f78b0555a6733ffbf6ab98bd#A#compaction#27 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-16T09:58:08,508 DEBUG [RS:0;037c58a19ddb:37267-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/d9e28756f78b0555a6733ffbf6ab98bd/.tmp/A/a919ab9e3a914bc59f13fe1f3f7b251e is 50, key is test_row_0/A:col10/1731751087270/Put/seqid=0 2024-11-16T09:58:08,512 INFO [RS:0;037c58a19ddb:37267-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): d9e28756f78b0555a6733ffbf6ab98bd#B#compaction#28 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-16T09:58:08,513 DEBUG [RS:0;037c58a19ddb:37267-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/d9e28756f78b0555a6733ffbf6ab98bd/.tmp/B/706a630aaba4416ba4ad59882a64793c is 50, key is test_row_0/B:col10/1731751087270/Put/seqid=0 2024-11-16T09:58:08,548 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37591 is added to blk_1073741865_1041 (size=12459) 2024-11-16T09:58:08,550 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37591 is added to blk_1073741864_1040 (size=12459) 2024-11-16T09:58:08,565 DEBUG [RS:0;037c58a19ddb:37267-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/d9e28756f78b0555a6733ffbf6ab98bd/.tmp/B/706a630aaba4416ba4ad59882a64793c as hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/d9e28756f78b0555a6733ffbf6ab98bd/B/706a630aaba4416ba4ad59882a64793c 2024-11-16T09:58:08,568 DEBUG [RS:0;037c58a19ddb:37267-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/d9e28756f78b0555a6733ffbf6ab98bd/.tmp/A/a919ab9e3a914bc59f13fe1f3f7b251e as hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/d9e28756f78b0555a6733ffbf6ab98bd/A/a919ab9e3a914bc59f13fe1f3f7b251e 2024-11-16T09:58:08,577 INFO [RS:0;037c58a19ddb:37267-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in d9e28756f78b0555a6733ffbf6ab98bd/B of d9e28756f78b0555a6733ffbf6ab98bd into 706a630aaba4416ba4ad59882a64793c(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-16T09:58:08,577 DEBUG [RS:0;037c58a19ddb:37267-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for d9e28756f78b0555a6733ffbf6ab98bd: 2024-11-16T09:58:08,577 INFO [RS:0;037c58a19ddb:37267-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731751082172.d9e28756f78b0555a6733ffbf6ab98bd., storeName=d9e28756f78b0555a6733ffbf6ab98bd/B, priority=13, startTime=1731751088480; duration=0sec 2024-11-16T09:58:08,578 DEBUG [RS:0;037c58a19ddb:37267-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-16T09:58:08,578 DEBUG [RS:0;037c58a19ddb:37267-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: d9e28756f78b0555a6733ffbf6ab98bd:B 2024-11-16T09:58:08,578 DEBUG [RS:0;037c58a19ddb:37267-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-16T09:58:08,581 DEBUG [RS:0;037c58a19ddb:37267-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36359 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-16T09:58:08,581 DEBUG [RS:0;037c58a19ddb:37267-longCompactions-0 {}] regionserver.HStore(1541): d9e28756f78b0555a6733ffbf6ab98bd/C is initiating minor compaction (all files) 2024-11-16T09:58:08,581 INFO [RS:0;037c58a19ddb:37267-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of d9e28756f78b0555a6733ffbf6ab98bd/C in TestAcidGuarantees,,1731751082172.d9e28756f78b0555a6733ffbf6ab98bd. 2024-11-16T09:58:08,581 INFO [RS:0;037c58a19ddb:37267-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/d9e28756f78b0555a6733ffbf6ab98bd/C/aba8c9e7079440bebfb145d1604cd540, hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/d9e28756f78b0555a6733ffbf6ab98bd/C/775aa7ebdf894f119f74fcbc7d28ea62, hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/d9e28756f78b0555a6733ffbf6ab98bd/C/43f98f5a3e8c418080fd6b081c369f71] into tmpdir=hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/d9e28756f78b0555a6733ffbf6ab98bd/.tmp, totalSize=35.5 K 2024-11-16T09:58:08,582 DEBUG [RS:0;037c58a19ddb:37267-longCompactions-0 {}] compactions.Compactor(225): Compacting aba8c9e7079440bebfb145d1604cd540, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=96, earliestPutTs=1731751086150 2024-11-16T09:58:08,583 DEBUG [RS:0;037c58a19ddb:37267-longCompactions-0 {}] compactions.Compactor(225): Compacting 775aa7ebdf894f119f74fcbc7d28ea62, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=119, earliestPutTs=1731751086855 2024-11-16T09:58:08,583 INFO [RS:0;037c58a19ddb:37267-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in d9e28756f78b0555a6733ffbf6ab98bd/A of d9e28756f78b0555a6733ffbf6ab98bd into a919ab9e3a914bc59f13fe1f3f7b251e(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-16T09:58:08,584 DEBUG [RS:0;037c58a19ddb:37267-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for d9e28756f78b0555a6733ffbf6ab98bd: 2024-11-16T09:58:08,584 INFO [RS:0;037c58a19ddb:37267-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731751082172.d9e28756f78b0555a6733ffbf6ab98bd., storeName=d9e28756f78b0555a6733ffbf6ab98bd/A, priority=13, startTime=1731751088480; duration=0sec 2024-11-16T09:58:08,584 DEBUG [RS:0;037c58a19ddb:37267-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-16T09:58:08,584 DEBUG [RS:0;037c58a19ddb:37267-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: d9e28756f78b0555a6733ffbf6ab98bd:A 2024-11-16T09:58:08,586 DEBUG [RS:0;037c58a19ddb:37267-longCompactions-0 {}] compactions.Compactor(225): Compacting 43f98f5a3e8c418080fd6b081c369f71, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=137, earliestPutTs=1731751087270 2024-11-16T09:58:08,603 INFO [RS:0;037c58a19ddb:37267-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): d9e28756f78b0555a6733ffbf6ab98bd#C#compaction#29 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-16T09:58:08,604 DEBUG [RS:0;037c58a19ddb:37267-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/d9e28756f78b0555a6733ffbf6ab98bd/.tmp/C/7a3f8766b7834cda80087a8ec71ec9e7 is 50, key is test_row_0/C:col10/1731751087270/Put/seqid=0 2024-11-16T09:58:08,622 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37591 is added to blk_1073741866_1042 (size=12459) 2024-11-16T09:58:08,623 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37877 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=7 2024-11-16T09:58:08,624 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-16T09:58:08,626 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37877 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-16T09:58:08,628 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37877 {}] procedure2.ProcedureExecutor(1139): Stored pid=9, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=9, table=TestAcidGuarantees 2024-11-16T09:58:08,631 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=9, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=9, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-16T09:58:08,632 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37877 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=9 2024-11-16T09:58:08,632 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=9, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=9, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-16T09:58:08,633 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=10, ppid=9, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-16T09:58:08,641 DEBUG [RS:0;037c58a19ddb:37267-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/d9e28756f78b0555a6733ffbf6ab98bd/.tmp/C/7a3f8766b7834cda80087a8ec71ec9e7 as hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/d9e28756f78b0555a6733ffbf6ab98bd/C/7a3f8766b7834cda80087a8ec71ec9e7 2024-11-16T09:58:08,652 INFO [RS:0;037c58a19ddb:37267-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in d9e28756f78b0555a6733ffbf6ab98bd/C of d9e28756f78b0555a6733ffbf6ab98bd into 7a3f8766b7834cda80087a8ec71ec9e7(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-16T09:58:08,652 DEBUG [RS:0;037c58a19ddb:37267-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for d9e28756f78b0555a6733ffbf6ab98bd: 2024-11-16T09:58:08,652 INFO [RS:0;037c58a19ddb:37267-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731751082172.d9e28756f78b0555a6733ffbf6ab98bd., storeName=d9e28756f78b0555a6733ffbf6ab98bd/C, priority=13, startTime=1731751088480; duration=0sec 2024-11-16T09:58:08,652 DEBUG [RS:0;037c58a19ddb:37267-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-16T09:58:08,652 DEBUG [RS:0;037c58a19ddb:37267-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: d9e28756f78b0555a6733ffbf6ab98bd:C 2024-11-16T09:58:08,677 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] regionserver.HRegion(8855): Flush requested on d9e28756f78b0555a6733ffbf6ab98bd 2024-11-16T09:58:08,677 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-16T09:58:08,677 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing d9e28756f78b0555a6733ffbf6ab98bd 3/3 column families, dataSize=127.47 KB heapSize=334.73 KB 2024-11-16T09:58:08,688 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/d9e28756f78b0555a6733ffbf6ab98bd/.tmp/A/8900f6b9a93d4df2b2b02e97cb83983f is 50, key is test_row_0/A:col10/1731751088677/Put/seqid=0 2024-11-16T09:58:08,698 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37591 is added to blk_1073741867_1043 (size=14541) 2024-11-16T09:58:08,743 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37877 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=9 2024-11-16T09:58:08,750 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9e28756f78b0555a6733ffbf6ab98bd, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T09:58:08,750 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9e28756f78b0555a6733ffbf6ab98bd, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T09:58:08,750 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9e28756f78b0555a6733ffbf6ab98bd, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T09:58:08,750 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39696 deadline: 1731751148742, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9e28756f78b0555a6733ffbf6ab98bd, server=037c58a19ddb,37267,1731751077844 2024-11-16T09:58:08,750 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39620 deadline: 1731751148740, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9e28756f78b0555a6733ffbf6ab98bd, server=037c58a19ddb,37267,1731751077844 2024-11-16T09:58:08,750 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39654 deadline: 1731751148743, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9e28756f78b0555a6733ffbf6ab98bd, server=037c58a19ddb,37267,1731751077844 2024-11-16T09:58:08,752 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731751082172.d9e28756f78b0555a6733ffbf6ab98bd., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 , the old value is region=TestAcidGuarantees,,1731751082172.d9e28756f78b0555a6733ffbf6ab98bd., hostname=037c58a19ddb,37267,1731751077844, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9e28756f78b0555a6733ffbf6ab98bd, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:58:08,752 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731751082172.d9e28756f78b0555a6733ffbf6ab98bd., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 , the old value is region=TestAcidGuarantees,,1731751082172.d9e28756f78b0555a6733ffbf6ab98bd., hostname=037c58a19ddb,37267,1731751077844, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9e28756f78b0555a6733ffbf6ab98bd, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:58:08,752 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731751082172.d9e28756f78b0555a6733ffbf6ab98bd., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9e28756f78b0555a6733ffbf6ab98bd, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:58:08,752 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731751082172.d9e28756f78b0555a6733ffbf6ab98bd., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9e28756f78b0555a6733ffbf6ab98bd, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:58:08,752 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731751082172.d9e28756f78b0555a6733ffbf6ab98bd., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 because the exception is null or not the one we care about 2024-11-16T09:58:08,752 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731751082172.d9e28756f78b0555a6733ffbf6ab98bd., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 because the exception is null or not the one we care about 2024-11-16T09:58:08,753 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731751082172.d9e28756f78b0555a6733ffbf6ab98bd., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 , the old value is region=TestAcidGuarantees,,1731751082172.d9e28756f78b0555a6733ffbf6ab98bd., hostname=037c58a19ddb,37267,1731751077844, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9e28756f78b0555a6733ffbf6ab98bd, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:58:08,753 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731751082172.d9e28756f78b0555a6733ffbf6ab98bd., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9e28756f78b0555a6733ffbf6ab98bd, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:58:08,753 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731751082172.d9e28756f78b0555a6733ffbf6ab98bd., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 because the exception is null or not the one we care about 2024-11-16T09:58:08,753 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9e28756f78b0555a6733ffbf6ab98bd, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T09:58:08,754 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39638 deadline: 1731751148751, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9e28756f78b0555a6733ffbf6ab98bd, server=037c58a19ddb,37267,1731751077844 2024-11-16T09:58:08,754 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731751082172.d9e28756f78b0555a6733ffbf6ab98bd., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 , the old value is region=TestAcidGuarantees,,1731751082172.d9e28756f78b0555a6733ffbf6ab98bd., hostname=037c58a19ddb,37267,1731751077844, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9e28756f78b0555a6733ffbf6ab98bd, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:58:08,754 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731751082172.d9e28756f78b0555a6733ffbf6ab98bd., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9e28756f78b0555a6733ffbf6ab98bd, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:58:08,754 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9e28756f78b0555a6733ffbf6ab98bd, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T09:58:08,754 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731751082172.d9e28756f78b0555a6733ffbf6ab98bd., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 because the exception is null or not the one we care about 2024-11-16T09:58:08,754 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39630 deadline: 1731751148753, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9e28756f78b0555a6733ffbf6ab98bd, server=037c58a19ddb,37267,1731751077844 2024-11-16T09:58:08,755 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731751082172.d9e28756f78b0555a6733ffbf6ab98bd., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 , the old value is region=TestAcidGuarantees,,1731751082172.d9e28756f78b0555a6733ffbf6ab98bd., hostname=037c58a19ddb,37267,1731751077844, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9e28756f78b0555a6733ffbf6ab98bd, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:58:08,755 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731751082172.d9e28756f78b0555a6733ffbf6ab98bd., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9e28756f78b0555a6733ffbf6ab98bd, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:58:08,755 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731751082172.d9e28756f78b0555a6733ffbf6ab98bd., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 because the exception is null or not the one we care about 2024-11-16T09:58:08,792 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=37267 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=10 2024-11-16T09:58:08,792 DEBUG [RS_FLUSH_OPERATIONS-regionserver/037c58a19ddb:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731751082172.d9e28756f78b0555a6733ffbf6ab98bd. 2024-11-16T09:58:08,792 DEBUG [RS_FLUSH_OPERATIONS-regionserver/037c58a19ddb:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731751082172.d9e28756f78b0555a6733ffbf6ab98bd. as already flushing 2024-11-16T09:58:08,792 DEBUG [RS_FLUSH_OPERATIONS-regionserver/037c58a19ddb:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731751082172.d9e28756f78b0555a6733ffbf6ab98bd. 2024-11-16T09:58:08,793 ERROR [RS_FLUSH_OPERATIONS-regionserver/037c58a19ddb:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] handler.RSProcedureHandler(58): pid=10 java.io.IOException: Unable to complete flush {ENCODED => d9e28756f78b0555a6733ffbf6ab98bd, NAME => 'TestAcidGuarantees,,1731751082172.d9e28756f78b0555a6733ffbf6ab98bd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T09:58:08,793 DEBUG [RS_FLUSH_OPERATIONS-regionserver/037c58a19ddb:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=10 java.io.IOException: Unable to complete flush {ENCODED => d9e28756f78b0555a6733ffbf6ab98bd, NAME => 'TestAcidGuarantees,,1731751082172.d9e28756f78b0555a6733ffbf6ab98bd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T09:58:08,794 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37877 {}] master.HMaster(4177): Remote procedure failed, pid=10 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d9e28756f78b0555a6733ffbf6ab98bd, NAME => 'TestAcidGuarantees,,1731751082172.d9e28756f78b0555a6733ffbf6ab98bd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d9e28756f78b0555a6733ffbf6ab98bd, NAME => 'TestAcidGuarantees,,1731751082172.d9e28756f78b0555a6733ffbf6ab98bd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T09:58:08,864 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9e28756f78b0555a6733ffbf6ab98bd, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T09:58:08,864 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39696 deadline: 1731751148863, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9e28756f78b0555a6733ffbf6ab98bd, server=037c58a19ddb,37267,1731751077844 2024-11-16T09:58:08,864 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9e28756f78b0555a6733ffbf6ab98bd, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T09:58:08,865 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39638 deadline: 1731751148864, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9e28756f78b0555a6733ffbf6ab98bd, server=037c58a19ddb,37267,1731751077844 2024-11-16T09:58:08,865 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731751082172.d9e28756f78b0555a6733ffbf6ab98bd., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 , the old value is region=TestAcidGuarantees,,1731751082172.d9e28756f78b0555a6733ffbf6ab98bd., hostname=037c58a19ddb,37267,1731751077844, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9e28756f78b0555a6733ffbf6ab98bd, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:58:08,865 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9e28756f78b0555a6733ffbf6ab98bd, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T09:58:08,865 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731751082172.d9e28756f78b0555a6733ffbf6ab98bd., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9e28756f78b0555a6733ffbf6ab98bd, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:58:08,865 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731751082172.d9e28756f78b0555a6733ffbf6ab98bd., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 because the exception is null or not the one we care about 2024-11-16T09:58:08,865 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39620 deadline: 1731751148864, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9e28756f78b0555a6733ffbf6ab98bd, server=037c58a19ddb,37267,1731751077844 2024-11-16T09:58:08,865 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731751082172.d9e28756f78b0555a6733ffbf6ab98bd., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 , the old value is region=TestAcidGuarantees,,1731751082172.d9e28756f78b0555a6733ffbf6ab98bd., hostname=037c58a19ddb,37267,1731751077844, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9e28756f78b0555a6733ffbf6ab98bd, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:58:08,865 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731751082172.d9e28756f78b0555a6733ffbf6ab98bd., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9e28756f78b0555a6733ffbf6ab98bd, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:58:08,865 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731751082172.d9e28756f78b0555a6733ffbf6ab98bd., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 because the exception is null or not the one we care about 2024-11-16T09:58:08,865 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731751082172.d9e28756f78b0555a6733ffbf6ab98bd., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 , the old value is region=TestAcidGuarantees,,1731751082172.d9e28756f78b0555a6733ffbf6ab98bd., hostname=037c58a19ddb,37267,1731751077844, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9e28756f78b0555a6733ffbf6ab98bd, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:58:08,865 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731751082172.d9e28756f78b0555a6733ffbf6ab98bd., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9e28756f78b0555a6733ffbf6ab98bd, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:58:08,865 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9e28756f78b0555a6733ffbf6ab98bd, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T09:58:08,865 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731751082172.d9e28756f78b0555a6733ffbf6ab98bd., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 because the exception is null or not the one we care about 2024-11-16T09:58:08,866 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39654 deadline: 1731751148864, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9e28756f78b0555a6733ffbf6ab98bd, server=037c58a19ddb,37267,1731751077844 2024-11-16T09:58:08,866 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731751082172.d9e28756f78b0555a6733ffbf6ab98bd., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 , the old value is region=TestAcidGuarantees,,1731751082172.d9e28756f78b0555a6733ffbf6ab98bd., hostname=037c58a19ddb,37267,1731751077844, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9e28756f78b0555a6733ffbf6ab98bd, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:58:08,867 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731751082172.d9e28756f78b0555a6733ffbf6ab98bd., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9e28756f78b0555a6733ffbf6ab98bd, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:58:08,867 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731751082172.d9e28756f78b0555a6733ffbf6ab98bd., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 because the exception is null or not the one we care about 2024-11-16T09:58:08,868 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9e28756f78b0555a6733ffbf6ab98bd, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T09:58:08,868 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39630 deadline: 1731751148865, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9e28756f78b0555a6733ffbf6ab98bd, server=037c58a19ddb,37267,1731751077844 2024-11-16T09:58:08,868 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731751082172.d9e28756f78b0555a6733ffbf6ab98bd., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 , the old value is region=TestAcidGuarantees,,1731751082172.d9e28756f78b0555a6733ffbf6ab98bd., hostname=037c58a19ddb,37267,1731751077844, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9e28756f78b0555a6733ffbf6ab98bd, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:58:08,869 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731751082172.d9e28756f78b0555a6733ffbf6ab98bd., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9e28756f78b0555a6733ffbf6ab98bd, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:58:08,869 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731751082172.d9e28756f78b0555a6733ffbf6ab98bd., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 because the exception is null or not the one we care about 2024-11-16T09:58:08,947 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=37267 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=10 2024-11-16T09:58:08,948 DEBUG [RS_FLUSH_OPERATIONS-regionserver/037c58a19ddb:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731751082172.d9e28756f78b0555a6733ffbf6ab98bd. 2024-11-16T09:58:08,948 DEBUG [RS_FLUSH_OPERATIONS-regionserver/037c58a19ddb:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731751082172.d9e28756f78b0555a6733ffbf6ab98bd. as already flushing 2024-11-16T09:58:08,948 DEBUG [RS_FLUSH_OPERATIONS-regionserver/037c58a19ddb:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731751082172.d9e28756f78b0555a6733ffbf6ab98bd. 2024-11-16T09:58:08,948 ERROR [RS_FLUSH_OPERATIONS-regionserver/037c58a19ddb:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] handler.RSProcedureHandler(58): pid=10 java.io.IOException: Unable to complete flush {ENCODED => d9e28756f78b0555a6733ffbf6ab98bd, NAME => 'TestAcidGuarantees,,1731751082172.d9e28756f78b0555a6733ffbf6ab98bd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T09:58:08,948 DEBUG [RS_FLUSH_OPERATIONS-regionserver/037c58a19ddb:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=10 java.io.IOException: Unable to complete flush {ENCODED => d9e28756f78b0555a6733ffbf6ab98bd, NAME => 'TestAcidGuarantees,,1731751082172.d9e28756f78b0555a6733ffbf6ab98bd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T09:58:08,949 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37877 {}] master.HMaster(4177): Remote procedure failed, pid=10 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d9e28756f78b0555a6733ffbf6ab98bd, NAME => 'TestAcidGuarantees,,1731751082172.d9e28756f78b0555a6733ffbf6ab98bd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d9e28756f78b0555a6733ffbf6ab98bd, NAME => 'TestAcidGuarantees,,1731751082172.d9e28756f78b0555a6733ffbf6ab98bd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T09:58:08,953 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37877 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=9 2024-11-16T09:58:09,073 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9e28756f78b0555a6733ffbf6ab98bd, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T09:58:09,074 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39696 deadline: 1731751149073, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9e28756f78b0555a6733ffbf6ab98bd, server=037c58a19ddb,37267,1731751077844 2024-11-16T09:58:09,074 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9e28756f78b0555a6733ffbf6ab98bd, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T09:58:09,074 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731751082172.d9e28756f78b0555a6733ffbf6ab98bd., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 , the old value is region=TestAcidGuarantees,,1731751082172.d9e28756f78b0555a6733ffbf6ab98bd., hostname=037c58a19ddb,37267,1731751077844, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9e28756f78b0555a6733ffbf6ab98bd, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:58:09,075 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731751082172.d9e28756f78b0555a6733ffbf6ab98bd., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9e28756f78b0555a6733ffbf6ab98bd, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:58:09,075 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731751082172.d9e28756f78b0555a6733ffbf6ab98bd., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 because the exception is null or not the one we care about 2024-11-16T09:58:09,075 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39638 deadline: 1731751149073, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9e28756f78b0555a6733ffbf6ab98bd, server=037c58a19ddb,37267,1731751077844 2024-11-16T09:58:09,075 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9e28756f78b0555a6733ffbf6ab98bd, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T09:58:09,075 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731751082172.d9e28756f78b0555a6733ffbf6ab98bd., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 , the old value is region=TestAcidGuarantees,,1731751082172.d9e28756f78b0555a6733ffbf6ab98bd., hostname=037c58a19ddb,37267,1731751077844, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9e28756f78b0555a6733ffbf6ab98bd, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:58:09,075 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731751082172.d9e28756f78b0555a6733ffbf6ab98bd., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9e28756f78b0555a6733ffbf6ab98bd, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:58:09,075 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39654 deadline: 1731751149074, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9e28756f78b0555a6733ffbf6ab98bd, server=037c58a19ddb,37267,1731751077844 2024-11-16T09:58:09,075 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731751082172.d9e28756f78b0555a6733ffbf6ab98bd., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 because the exception is null or not the one we care about 2024-11-16T09:58:09,075 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731751082172.d9e28756f78b0555a6733ffbf6ab98bd., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 , the old value is region=TestAcidGuarantees,,1731751082172.d9e28756f78b0555a6733ffbf6ab98bd., hostname=037c58a19ddb,37267,1731751077844, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9e28756f78b0555a6733ffbf6ab98bd, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:58:09,075 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9e28756f78b0555a6733ffbf6ab98bd, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T09:58:09,075 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731751082172.d9e28756f78b0555a6733ffbf6ab98bd., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9e28756f78b0555a6733ffbf6ab98bd, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:58:09,075 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731751082172.d9e28756f78b0555a6733ffbf6ab98bd., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 because the exception is null or not the one we care about 2024-11-16T09:58:09,076 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39620 deadline: 1731751149074, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9e28756f78b0555a6733ffbf6ab98bd, server=037c58a19ddb,37267,1731751077844 2024-11-16T09:58:09,076 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731751082172.d9e28756f78b0555a6733ffbf6ab98bd., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 , the old value is region=TestAcidGuarantees,,1731751082172.d9e28756f78b0555a6733ffbf6ab98bd., hostname=037c58a19ddb,37267,1731751077844, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9e28756f78b0555a6733ffbf6ab98bd, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:58:09,076 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731751082172.d9e28756f78b0555a6733ffbf6ab98bd., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9e28756f78b0555a6733ffbf6ab98bd, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:58:09,076 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731751082172.d9e28756f78b0555a6733ffbf6ab98bd., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 because the exception is null or not the one we care about 2024-11-16T09:58:09,078 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9e28756f78b0555a6733ffbf6ab98bd, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T09:58:09,078 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39630 deadline: 1731751149075, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9e28756f78b0555a6733ffbf6ab98bd, server=037c58a19ddb,37267,1731751077844 2024-11-16T09:58:09,079 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731751082172.d9e28756f78b0555a6733ffbf6ab98bd., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 , the old value is region=TestAcidGuarantees,,1731751082172.d9e28756f78b0555a6733ffbf6ab98bd., hostname=037c58a19ddb,37267,1731751077844, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9e28756f78b0555a6733ffbf6ab98bd, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:58:09,079 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731751082172.d9e28756f78b0555a6733ffbf6ab98bd., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9e28756f78b0555a6733ffbf6ab98bd, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:58:09,079 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731751082172.d9e28756f78b0555a6733ffbf6ab98bd., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 because the exception is null or not the one we care about 2024-11-16T09:58:09,099 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=164 (bloomFilter=true), to=hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/d9e28756f78b0555a6733ffbf6ab98bd/.tmp/A/8900f6b9a93d4df2b2b02e97cb83983f 2024-11-16T09:58:09,102 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=37267 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=10 2024-11-16T09:58:09,103 DEBUG [RS_FLUSH_OPERATIONS-regionserver/037c58a19ddb:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731751082172.d9e28756f78b0555a6733ffbf6ab98bd. 2024-11-16T09:58:09,103 DEBUG [RS_FLUSH_OPERATIONS-regionserver/037c58a19ddb:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731751082172.d9e28756f78b0555a6733ffbf6ab98bd. as already flushing 2024-11-16T09:58:09,103 DEBUG [RS_FLUSH_OPERATIONS-regionserver/037c58a19ddb:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731751082172.d9e28756f78b0555a6733ffbf6ab98bd. 2024-11-16T09:58:09,103 ERROR [RS_FLUSH_OPERATIONS-regionserver/037c58a19ddb:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] handler.RSProcedureHandler(58): pid=10 java.io.IOException: Unable to complete flush {ENCODED => d9e28756f78b0555a6733ffbf6ab98bd, NAME => 'TestAcidGuarantees,,1731751082172.d9e28756f78b0555a6733ffbf6ab98bd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T09:58:09,103 DEBUG [RS_FLUSH_OPERATIONS-regionserver/037c58a19ddb:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=10 java.io.IOException: Unable to complete flush {ENCODED => d9e28756f78b0555a6733ffbf6ab98bd, NAME => 'TestAcidGuarantees,,1731751082172.d9e28756f78b0555a6733ffbf6ab98bd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T09:58:09,104 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37877 {}] master.HMaster(4177): Remote procedure failed, pid=10 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d9e28756f78b0555a6733ffbf6ab98bd, NAME => 'TestAcidGuarantees,,1731751082172.d9e28756f78b0555a6733ffbf6ab98bd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d9e28756f78b0555a6733ffbf6ab98bd, NAME => 'TestAcidGuarantees,,1731751082172.d9e28756f78b0555a6733ffbf6ab98bd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T09:58:09,114 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/d9e28756f78b0555a6733ffbf6ab98bd/.tmp/B/ea425dc15ba4426e82f075375750f365 is 50, key is test_row_0/B:col10/1731751088677/Put/seqid=0 2024-11-16T09:58:09,128 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37591 is added to blk_1073741868_1044 (size=12151) 2024-11-16T09:58:09,130 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=164 (bloomFilter=true), to=hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/d9e28756f78b0555a6733ffbf6ab98bd/.tmp/B/ea425dc15ba4426e82f075375750f365 2024-11-16T09:58:09,148 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/d9e28756f78b0555a6733ffbf6ab98bd/.tmp/C/bc833c50bb2540ac9dda563506ca1e2a is 50, key is test_row_0/C:col10/1731751088677/Put/seqid=0 2024-11-16T09:58:09,162 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37591 is added to blk_1073741869_1045 (size=12151) 2024-11-16T09:58:09,258 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=37267 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=10 2024-11-16T09:58:09,258 DEBUG [RS_FLUSH_OPERATIONS-regionserver/037c58a19ddb:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731751082172.d9e28756f78b0555a6733ffbf6ab98bd. 2024-11-16T09:58:09,258 DEBUG [RS_FLUSH_OPERATIONS-regionserver/037c58a19ddb:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731751082172.d9e28756f78b0555a6733ffbf6ab98bd. as already flushing 2024-11-16T09:58:09,259 DEBUG [RS_FLUSH_OPERATIONS-regionserver/037c58a19ddb:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731751082172.d9e28756f78b0555a6733ffbf6ab98bd. 2024-11-16T09:58:09,259 ERROR [RS_FLUSH_OPERATIONS-regionserver/037c58a19ddb:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] handler.RSProcedureHandler(58): pid=10 java.io.IOException: Unable to complete flush {ENCODED => d9e28756f78b0555a6733ffbf6ab98bd, NAME => 'TestAcidGuarantees,,1731751082172.d9e28756f78b0555a6733ffbf6ab98bd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T09:58:09,259 DEBUG [RS_FLUSH_OPERATIONS-regionserver/037c58a19ddb:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=10 java.io.IOException: Unable to complete flush {ENCODED => d9e28756f78b0555a6733ffbf6ab98bd, NAME => 'TestAcidGuarantees,,1731751082172.d9e28756f78b0555a6733ffbf6ab98bd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T09:58:09,260 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37877 {}] master.HMaster(4177): Remote procedure failed, pid=10 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d9e28756f78b0555a6733ffbf6ab98bd, NAME => 'TestAcidGuarantees,,1731751082172.d9e28756f78b0555a6733ffbf6ab98bd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d9e28756f78b0555a6733ffbf6ab98bd, NAME => 'TestAcidGuarantees,,1731751082172.d9e28756f78b0555a6733ffbf6ab98bd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T09:58:09,263 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37877 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=9 2024-11-16T09:58:09,383 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9e28756f78b0555a6733ffbf6ab98bd, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T09:58:09,384 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39696 deadline: 1731751149383, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9e28756f78b0555a6733ffbf6ab98bd, server=037c58a19ddb,37267,1731751077844 2024-11-16T09:58:09,384 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731751082172.d9e28756f78b0555a6733ffbf6ab98bd., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 , the old value is region=TestAcidGuarantees,,1731751082172.d9e28756f78b0555a6733ffbf6ab98bd., hostname=037c58a19ddb,37267,1731751077844, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9e28756f78b0555a6733ffbf6ab98bd, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:58:09,384 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9e28756f78b0555a6733ffbf6ab98bd, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T09:58:09,384 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731751082172.d9e28756f78b0555a6733ffbf6ab98bd., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9e28756f78b0555a6733ffbf6ab98bd, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:58:09,384 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731751082172.d9e28756f78b0555a6733ffbf6ab98bd., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 because the exception is null or not the one we care about 2024-11-16T09:58:09,384 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39638 deadline: 1731751149383, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9e28756f78b0555a6733ffbf6ab98bd, server=037c58a19ddb,37267,1731751077844 2024-11-16T09:58:09,385 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731751082172.d9e28756f78b0555a6733ffbf6ab98bd., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 , the old value is region=TestAcidGuarantees,,1731751082172.d9e28756f78b0555a6733ffbf6ab98bd., hostname=037c58a19ddb,37267,1731751077844, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9e28756f78b0555a6733ffbf6ab98bd, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:58:09,385 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9e28756f78b0555a6733ffbf6ab98bd, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T09:58:09,385 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731751082172.d9e28756f78b0555a6733ffbf6ab98bd., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9e28756f78b0555a6733ffbf6ab98bd, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:58:09,385 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731751082172.d9e28756f78b0555a6733ffbf6ab98bd., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 because the exception is null or not the one we care about 2024-11-16T09:58:09,385 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39654 deadline: 1731751149383, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9e28756f78b0555a6733ffbf6ab98bd, server=037c58a19ddb,37267,1731751077844 2024-11-16T09:58:09,386 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731751082172.d9e28756f78b0555a6733ffbf6ab98bd., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 , the old value is region=TestAcidGuarantees,,1731751082172.d9e28756f78b0555a6733ffbf6ab98bd., hostname=037c58a19ddb,37267,1731751077844, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9e28756f78b0555a6733ffbf6ab98bd, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:58:09,386 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731751082172.d9e28756f78b0555a6733ffbf6ab98bd., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9e28756f78b0555a6733ffbf6ab98bd, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:58:09,386 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9e28756f78b0555a6733ffbf6ab98bd, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T09:58:09,386 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731751082172.d9e28756f78b0555a6733ffbf6ab98bd., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 because the exception is null or not the one we care about 2024-11-16T09:58:09,386 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39620 deadline: 1731751149383, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9e28756f78b0555a6733ffbf6ab98bd, server=037c58a19ddb,37267,1731751077844 2024-11-16T09:58:09,386 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731751082172.d9e28756f78b0555a6733ffbf6ab98bd., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 , the old value is region=TestAcidGuarantees,,1731751082172.d9e28756f78b0555a6733ffbf6ab98bd., hostname=037c58a19ddb,37267,1731751077844, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9e28756f78b0555a6733ffbf6ab98bd, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:58:09,386 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9e28756f78b0555a6733ffbf6ab98bd, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T09:58:09,386 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731751082172.d9e28756f78b0555a6733ffbf6ab98bd., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9e28756f78b0555a6733ffbf6ab98bd, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:58:09,386 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731751082172.d9e28756f78b0555a6733ffbf6ab98bd., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 because the exception is null or not the one we care about 2024-11-16T09:58:09,387 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39630 deadline: 1731751149384, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9e28756f78b0555a6733ffbf6ab98bd, server=037c58a19ddb,37267,1731751077844 2024-11-16T09:58:09,387 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731751082172.d9e28756f78b0555a6733ffbf6ab98bd., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 , the old value is region=TestAcidGuarantees,,1731751082172.d9e28756f78b0555a6733ffbf6ab98bd., hostname=037c58a19ddb,37267,1731751077844, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9e28756f78b0555a6733ffbf6ab98bd, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:58:09,387 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731751082172.d9e28756f78b0555a6733ffbf6ab98bd., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9e28756f78b0555a6733ffbf6ab98bd, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:58:09,387 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731751082172.d9e28756f78b0555a6733ffbf6ab98bd., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 because the exception is null or not the one we care about 2024-11-16T09:58:09,413 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=37267 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=10 2024-11-16T09:58:09,414 DEBUG [RS_FLUSH_OPERATIONS-regionserver/037c58a19ddb:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731751082172.d9e28756f78b0555a6733ffbf6ab98bd. 2024-11-16T09:58:09,414 DEBUG [RS_FLUSH_OPERATIONS-regionserver/037c58a19ddb:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731751082172.d9e28756f78b0555a6733ffbf6ab98bd. as already flushing 2024-11-16T09:58:09,414 DEBUG [RS_FLUSH_OPERATIONS-regionserver/037c58a19ddb:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731751082172.d9e28756f78b0555a6733ffbf6ab98bd. 2024-11-16T09:58:09,414 ERROR [RS_FLUSH_OPERATIONS-regionserver/037c58a19ddb:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] handler.RSProcedureHandler(58): pid=10 java.io.IOException: Unable to complete flush {ENCODED => d9e28756f78b0555a6733ffbf6ab98bd, NAME => 'TestAcidGuarantees,,1731751082172.d9e28756f78b0555a6733ffbf6ab98bd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T09:58:09,414 DEBUG [RS_FLUSH_OPERATIONS-regionserver/037c58a19ddb:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=10 java.io.IOException: Unable to complete flush {ENCODED => d9e28756f78b0555a6733ffbf6ab98bd, NAME => 'TestAcidGuarantees,,1731751082172.d9e28756f78b0555a6733ffbf6ab98bd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T09:58:09,415 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37877 {}] master.HMaster(4177): Remote procedure failed, pid=10 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d9e28756f78b0555a6733ffbf6ab98bd, NAME => 'TestAcidGuarantees,,1731751082172.d9e28756f78b0555a6733ffbf6ab98bd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d9e28756f78b0555a6733ffbf6ab98bd, NAME => 'TestAcidGuarantees,,1731751082172.d9e28756f78b0555a6733ffbf6ab98bd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T09:58:09,563 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=164 (bloomFilter=true), to=hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/d9e28756f78b0555a6733ffbf6ab98bd/.tmp/C/bc833c50bb2540ac9dda563506ca1e2a 2024-11-16T09:58:09,570 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=37267 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=10 2024-11-16T09:58:09,570 DEBUG [RS_FLUSH_OPERATIONS-regionserver/037c58a19ddb:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731751082172.d9e28756f78b0555a6733ffbf6ab98bd. 2024-11-16T09:58:09,570 DEBUG [RS_FLUSH_OPERATIONS-regionserver/037c58a19ddb:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731751082172.d9e28756f78b0555a6733ffbf6ab98bd. as already flushing 2024-11-16T09:58:09,570 DEBUG [RS_FLUSH_OPERATIONS-regionserver/037c58a19ddb:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731751082172.d9e28756f78b0555a6733ffbf6ab98bd. 2024-11-16T09:58:09,570 ERROR [RS_FLUSH_OPERATIONS-regionserver/037c58a19ddb:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] handler.RSProcedureHandler(58): pid=10 java.io.IOException: Unable to complete flush {ENCODED => d9e28756f78b0555a6733ffbf6ab98bd, NAME => 'TestAcidGuarantees,,1731751082172.d9e28756f78b0555a6733ffbf6ab98bd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T09:58:09,571 DEBUG [RS_FLUSH_OPERATIONS-regionserver/037c58a19ddb:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=10 java.io.IOException: Unable to complete flush {ENCODED => d9e28756f78b0555a6733ffbf6ab98bd, NAME => 'TestAcidGuarantees,,1731751082172.d9e28756f78b0555a6733ffbf6ab98bd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T09:58:09,571 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37877 {}] master.HMaster(4177): Remote procedure failed, pid=10 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d9e28756f78b0555a6733ffbf6ab98bd, NAME => 'TestAcidGuarantees,,1731751082172.d9e28756f78b0555a6733ffbf6ab98bd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d9e28756f78b0555a6733ffbf6ab98bd, NAME => 'TestAcidGuarantees,,1731751082172.d9e28756f78b0555a6733ffbf6ab98bd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T09:58:09,575 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/d9e28756f78b0555a6733ffbf6ab98bd/.tmp/A/8900f6b9a93d4df2b2b02e97cb83983f as hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/d9e28756f78b0555a6733ffbf6ab98bd/A/8900f6b9a93d4df2b2b02e97cb83983f 2024-11-16T09:58:09,589 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/d9e28756f78b0555a6733ffbf6ab98bd/A/8900f6b9a93d4df2b2b02e97cb83983f, entries=200, sequenceid=164, filesize=14.2 K 2024-11-16T09:58:09,590 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/d9e28756f78b0555a6733ffbf6ab98bd/.tmp/B/ea425dc15ba4426e82f075375750f365 as hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/d9e28756f78b0555a6733ffbf6ab98bd/B/ea425dc15ba4426e82f075375750f365 2024-11-16T09:58:09,602 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/d9e28756f78b0555a6733ffbf6ab98bd/B/ea425dc15ba4426e82f075375750f365, entries=150, sequenceid=164, filesize=11.9 K 2024-11-16T09:58:09,605 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/d9e28756f78b0555a6733ffbf6ab98bd/.tmp/C/bc833c50bb2540ac9dda563506ca1e2a as hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/d9e28756f78b0555a6733ffbf6ab98bd/C/bc833c50bb2540ac9dda563506ca1e2a 2024-11-16T09:58:09,617 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/d9e28756f78b0555a6733ffbf6ab98bd/C/bc833c50bb2540ac9dda563506ca1e2a, entries=150, sequenceid=164, filesize=11.9 K 2024-11-16T09:58:09,620 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~140.89 KB/144270, heapSize ~369.84 KB/378720, currentSize=67.09 KB/68700 for d9e28756f78b0555a6733ffbf6ab98bd in 942ms, sequenceid=164, compaction requested=false 2024-11-16T09:58:09,620 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for d9e28756f78b0555a6733ffbf6ab98bd: 2024-11-16T09:58:09,727 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=37267 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=10 2024-11-16T09:58:09,727 DEBUG [RS_FLUSH_OPERATIONS-regionserver/037c58a19ddb:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731751082172.d9e28756f78b0555a6733ffbf6ab98bd. 2024-11-16T09:58:09,728 INFO [RS_FLUSH_OPERATIONS-regionserver/037c58a19ddb:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegion(2902): Flushing d9e28756f78b0555a6733ffbf6ab98bd 3/3 column families, dataSize=67.09 KB heapSize=176.53 KB 2024-11-16T09:58:09,736 DEBUG [RS_FLUSH_OPERATIONS-regionserver/037c58a19ddb:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/d9e28756f78b0555a6733ffbf6ab98bd/.tmp/A/5cd12febed204efabe434b45e92eccae is 50, key is test_row_0/A:col10/1731751088747/Put/seqid=0 2024-11-16T09:58:09,748 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37591 is added to blk_1073741870_1046 (size=12151) 2024-11-16T09:58:09,751 INFO [RS_FLUSH_OPERATIONS-regionserver/037c58a19ddb:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=177 (bloomFilter=true), to=hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/d9e28756f78b0555a6733ffbf6ab98bd/.tmp/A/5cd12febed204efabe434b45e92eccae 2024-11-16T09:58:09,766 DEBUG [RS_FLUSH_OPERATIONS-regionserver/037c58a19ddb:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/d9e28756f78b0555a6733ffbf6ab98bd/.tmp/B/15113ec6a9d84591914d35a99db57b4d is 50, key is test_row_0/B:col10/1731751088747/Put/seqid=0 2024-11-16T09:58:09,773 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37877 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=9 2024-11-16T09:58:09,776 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37591 is added to blk_1073741871_1047 (size=12151) 2024-11-16T09:58:09,781 INFO [RS_FLUSH_OPERATIONS-regionserver/037c58a19ddb:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=177 (bloomFilter=true), to=hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/d9e28756f78b0555a6733ffbf6ab98bd/.tmp/B/15113ec6a9d84591914d35a99db57b4d 2024-11-16T09:58:09,794 DEBUG [RS_FLUSH_OPERATIONS-regionserver/037c58a19ddb:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/d9e28756f78b0555a6733ffbf6ab98bd/.tmp/C/3ca5b8889378477bb2737c33c064ccc2 is 50, key is test_row_0/C:col10/1731751088747/Put/seqid=0 2024-11-16T09:58:09,822 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37591 is added to blk_1073741872_1048 (size=12151) 2024-11-16T09:58:09,824 INFO [RS_FLUSH_OPERATIONS-regionserver/037c58a19ddb:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=177 (bloomFilter=true), to=hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/d9e28756f78b0555a6733ffbf6ab98bd/.tmp/C/3ca5b8889378477bb2737c33c064ccc2 2024-11-16T09:58:09,835 DEBUG [RS_FLUSH_OPERATIONS-regionserver/037c58a19ddb:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/d9e28756f78b0555a6733ffbf6ab98bd/.tmp/A/5cd12febed204efabe434b45e92eccae as hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/d9e28756f78b0555a6733ffbf6ab98bd/A/5cd12febed204efabe434b45e92eccae 2024-11-16T09:58:09,845 INFO [RS_FLUSH_OPERATIONS-regionserver/037c58a19ddb:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/d9e28756f78b0555a6733ffbf6ab98bd/A/5cd12febed204efabe434b45e92eccae, entries=150, sequenceid=177, filesize=11.9 K 2024-11-16T09:58:09,848 DEBUG [RS_FLUSH_OPERATIONS-regionserver/037c58a19ddb:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/d9e28756f78b0555a6733ffbf6ab98bd/.tmp/B/15113ec6a9d84591914d35a99db57b4d as hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/d9e28756f78b0555a6733ffbf6ab98bd/B/15113ec6a9d84591914d35a99db57b4d 2024-11-16T09:58:09,859 INFO [RS_FLUSH_OPERATIONS-regionserver/037c58a19ddb:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/d9e28756f78b0555a6733ffbf6ab98bd/B/15113ec6a9d84591914d35a99db57b4d, entries=150, sequenceid=177, filesize=11.9 K 2024-11-16T09:58:09,863 DEBUG [RS_FLUSH_OPERATIONS-regionserver/037c58a19ddb:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/d9e28756f78b0555a6733ffbf6ab98bd/.tmp/C/3ca5b8889378477bb2737c33c064ccc2 as hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/d9e28756f78b0555a6733ffbf6ab98bd/C/3ca5b8889378477bb2737c33c064ccc2 2024-11-16T09:58:09,875 INFO [RS_FLUSH_OPERATIONS-regionserver/037c58a19ddb:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/d9e28756f78b0555a6733ffbf6ab98bd/C/3ca5b8889378477bb2737c33c064ccc2, entries=150, sequenceid=177, filesize=11.9 K 2024-11-16T09:58:09,878 INFO [RS_FLUSH_OPERATIONS-regionserver/037c58a19ddb:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegion(3140): Finished flush of dataSize ~67.09 KB/68700, heapSize ~176.48 KB/180720, currentSize=0 B/0 for d9e28756f78b0555a6733ffbf6ab98bd in 149ms, sequenceid=177, compaction requested=true 2024-11-16T09:58:09,878 DEBUG [RS_FLUSH_OPERATIONS-regionserver/037c58a19ddb:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegion(2603): Flush status journal for d9e28756f78b0555a6733ffbf6ab98bd: 2024-11-16T09:58:09,878 DEBUG [RS_FLUSH_OPERATIONS-regionserver/037c58a19ddb:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731751082172.d9e28756f78b0555a6733ffbf6ab98bd. 2024-11-16T09:58:09,878 DEBUG [RS_FLUSH_OPERATIONS-regionserver/037c58a19ddb:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=10 2024-11-16T09:58:09,879 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37877 {}] master.HMaster(4169): Remote procedure done, pid=10 2024-11-16T09:58:09,885 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=10, resume processing ppid=9 2024-11-16T09:58:09,885 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=10, ppid=9, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.2480 sec 2024-11-16T09:58:09,889 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=9, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=9, table=TestAcidGuarantees in 1.2600 sec 2024-11-16T09:58:09,918 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] regionserver.HRegion(8855): Flush requested on d9e28756f78b0555a6733ffbf6ab98bd 2024-11-16T09:58:09,919 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-16T09:58:09,919 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing d9e28756f78b0555a6733ffbf6ab98bd 3/3 column families, dataSize=53.67 KB heapSize=141.38 KB 2024-11-16T09:58:09,930 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/d9e28756f78b0555a6733ffbf6ab98bd/.tmp/A/0dbb7de1f967420a83f7452d01da47f7 is 50, key is test_row_0/A:col10/1731751089902/Put/seqid=0 2024-11-16T09:58:09,951 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37591 is added to blk_1073741873_1049 (size=19321) 2024-11-16T09:58:09,953 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=188 (bloomFilter=true), to=hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/d9e28756f78b0555a6733ffbf6ab98bd/.tmp/A/0dbb7de1f967420a83f7452d01da47f7 2024-11-16T09:58:09,968 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/d9e28756f78b0555a6733ffbf6ab98bd/.tmp/B/1239686f4cbd48f49490f5231be68ce7 is 50, key is test_row_0/B:col10/1731751089902/Put/seqid=0 2024-11-16T09:58:09,978 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37591 is added to blk_1073741874_1050 (size=12151) 2024-11-16T09:58:09,979 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=188 (bloomFilter=true), to=hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/d9e28756f78b0555a6733ffbf6ab98bd/.tmp/B/1239686f4cbd48f49490f5231be68ce7 2024-11-16T09:58:10,003 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/d9e28756f78b0555a6733ffbf6ab98bd/.tmp/C/2ae6fc50bfb0440580104fede526210d is 50, key is test_row_0/C:col10/1731751089902/Put/seqid=0 2024-11-16T09:58:10,018 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9e28756f78b0555a6733ffbf6ab98bd, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T09:58:10,019 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39620 deadline: 1731751149995, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9e28756f78b0555a6733ffbf6ab98bd, server=037c58a19ddb,37267,1731751077844 2024-11-16T09:58:10,019 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9e28756f78b0555a6733ffbf6ab98bd, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T09:58:10,020 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39696 deadline: 1731751150011, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9e28756f78b0555a6733ffbf6ab98bd, server=037c58a19ddb,37267,1731751077844 2024-11-16T09:58:10,020 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9e28756f78b0555a6733ffbf6ab98bd, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T09:58:10,020 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39638 deadline: 1731751150014, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9e28756f78b0555a6733ffbf6ab98bd, server=037c58a19ddb,37267,1731751077844 2024-11-16T09:58:10,021 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9e28756f78b0555a6733ffbf6ab98bd, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T09:58:10,021 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731751082172.d9e28756f78b0555a6733ffbf6ab98bd., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 , the old value is region=TestAcidGuarantees,,1731751082172.d9e28756f78b0555a6733ffbf6ab98bd., hostname=037c58a19ddb,37267,1731751077844, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9e28756f78b0555a6733ffbf6ab98bd, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:58:10,021 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731751082172.d9e28756f78b0555a6733ffbf6ab98bd., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9e28756f78b0555a6733ffbf6ab98bd, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:58:10,021 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731751082172.d9e28756f78b0555a6733ffbf6ab98bd., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 because the exception is null or not the one we care about 2024-11-16T09:58:10,022 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39654 deadline: 1731751150019, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9e28756f78b0555a6733ffbf6ab98bd, server=037c58a19ddb,37267,1731751077844 2024-11-16T09:58:10,022 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9e28756f78b0555a6733ffbf6ab98bd, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T09:58:10,022 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731751082172.d9e28756f78b0555a6733ffbf6ab98bd., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 , the old value is region=TestAcidGuarantees,,1731751082172.d9e28756f78b0555a6733ffbf6ab98bd., hostname=037c58a19ddb,37267,1731751077844, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9e28756f78b0555a6733ffbf6ab98bd, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:58:10,022 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731751082172.d9e28756f78b0555a6733ffbf6ab98bd., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 , the old value is region=TestAcidGuarantees,,1731751082172.d9e28756f78b0555a6733ffbf6ab98bd., hostname=037c58a19ddb,37267,1731751077844, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9e28756f78b0555a6733ffbf6ab98bd, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:58:10,022 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731751082172.d9e28756f78b0555a6733ffbf6ab98bd., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9e28756f78b0555a6733ffbf6ab98bd, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:58:10,022 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39630 deadline: 1731751150020, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9e28756f78b0555a6733ffbf6ab98bd, server=037c58a19ddb,37267,1731751077844 2024-11-16T09:58:10,022 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731751082172.d9e28756f78b0555a6733ffbf6ab98bd., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 because the exception is null or not the one we care about 2024-11-16T09:58:10,023 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731751082172.d9e28756f78b0555a6733ffbf6ab98bd., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 , the old value is region=TestAcidGuarantees,,1731751082172.d9e28756f78b0555a6733ffbf6ab98bd., hostname=037c58a19ddb,37267,1731751077844, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9e28756f78b0555a6733ffbf6ab98bd, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:58:10,023 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731751082172.d9e28756f78b0555a6733ffbf6ab98bd., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9e28756f78b0555a6733ffbf6ab98bd, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:58:10,023 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731751082172.d9e28756f78b0555a6733ffbf6ab98bd., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 because the exception is null or not the one we care about 2024-11-16T09:58:10,023 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731751082172.d9e28756f78b0555a6733ffbf6ab98bd., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 , the old value is region=TestAcidGuarantees,,1731751082172.d9e28756f78b0555a6733ffbf6ab98bd., hostname=037c58a19ddb,37267,1731751077844, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9e28756f78b0555a6733ffbf6ab98bd, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:58:10,023 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731751082172.d9e28756f78b0555a6733ffbf6ab98bd., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9e28756f78b0555a6733ffbf6ab98bd, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:58:10,022 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731751082172.d9e28756f78b0555a6733ffbf6ab98bd., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9e28756f78b0555a6733ffbf6ab98bd, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:58:10,023 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731751082172.d9e28756f78b0555a6733ffbf6ab98bd., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 because the exception is null or not the one we care about 2024-11-16T09:58:10,023 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731751082172.d9e28756f78b0555a6733ffbf6ab98bd., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 because the exception is null or not the one we care about 2024-11-16T09:58:10,031 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37591 is added to blk_1073741875_1051 (size=12151) 2024-11-16T09:58:10,033 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=188 (bloomFilter=true), to=hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/d9e28756f78b0555a6733ffbf6ab98bd/.tmp/C/2ae6fc50bfb0440580104fede526210d 2024-11-16T09:58:10,041 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/d9e28756f78b0555a6733ffbf6ab98bd/.tmp/A/0dbb7de1f967420a83f7452d01da47f7 as hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/d9e28756f78b0555a6733ffbf6ab98bd/A/0dbb7de1f967420a83f7452d01da47f7 2024-11-16T09:58:10,052 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/d9e28756f78b0555a6733ffbf6ab98bd/A/0dbb7de1f967420a83f7452d01da47f7, entries=300, sequenceid=188, filesize=18.9 K 2024-11-16T09:58:10,053 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/d9e28756f78b0555a6733ffbf6ab98bd/.tmp/B/1239686f4cbd48f49490f5231be68ce7 as hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/d9e28756f78b0555a6733ffbf6ab98bd/B/1239686f4cbd48f49490f5231be68ce7 2024-11-16T09:58:10,074 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/d9e28756f78b0555a6733ffbf6ab98bd/B/1239686f4cbd48f49490f5231be68ce7, entries=150, sequenceid=188, filesize=11.9 K 2024-11-16T09:58:10,076 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/d9e28756f78b0555a6733ffbf6ab98bd/.tmp/C/2ae6fc50bfb0440580104fede526210d as hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/d9e28756f78b0555a6733ffbf6ab98bd/C/2ae6fc50bfb0440580104fede526210d 2024-11-16T09:58:10,083 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/d9e28756f78b0555a6733ffbf6ab98bd/C/2ae6fc50bfb0440580104fede526210d, entries=150, sequenceid=188, filesize=11.9 K 2024-11-16T09:58:10,087 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~141.33 KB/144720, currentSize=147.60 KB/151140 for d9e28756f78b0555a6733ffbf6ab98bd in 168ms, sequenceid=188, compaction requested=true 2024-11-16T09:58:10,087 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for d9e28756f78b0555a6733ffbf6ab98bd: 2024-11-16T09:58:10,088 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store d9e28756f78b0555a6733ffbf6ab98bd:A, priority=-2147483648, current under compaction store size is 1 2024-11-16T09:58:10,088 DEBUG [RS:0;037c58a19ddb:37267-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-16T09:58:10,088 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-16T09:58:10,088 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store d9e28756f78b0555a6733ffbf6ab98bd:B, priority=-2147483648, current under compaction store size is 2 2024-11-16T09:58:10,088 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-16T09:58:10,088 DEBUG [RS:0;037c58a19ddb:37267-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-16T09:58:10,089 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store d9e28756f78b0555a6733ffbf6ab98bd:C, priority=-2147483648, current under compaction store size is 3 2024-11-16T09:58:10,089 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-16T09:58:10,093 DEBUG [RS:0;037c58a19ddb:37267-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48912 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-16T09:58:10,093 DEBUG [RS:0;037c58a19ddb:37267-longCompactions-0 {}] regionserver.HStore(1541): d9e28756f78b0555a6733ffbf6ab98bd/B is initiating minor compaction (all files) 2024-11-16T09:58:10,094 INFO [RS:0;037c58a19ddb:37267-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of d9e28756f78b0555a6733ffbf6ab98bd/B in TestAcidGuarantees,,1731751082172.d9e28756f78b0555a6733ffbf6ab98bd. 2024-11-16T09:58:10,094 INFO [RS:0;037c58a19ddb:37267-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/d9e28756f78b0555a6733ffbf6ab98bd/B/706a630aaba4416ba4ad59882a64793c, hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/d9e28756f78b0555a6733ffbf6ab98bd/B/ea425dc15ba4426e82f075375750f365, hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/d9e28756f78b0555a6733ffbf6ab98bd/B/15113ec6a9d84591914d35a99db57b4d, hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/d9e28756f78b0555a6733ffbf6ab98bd/B/1239686f4cbd48f49490f5231be68ce7] into tmpdir=hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/d9e28756f78b0555a6733ffbf6ab98bd/.tmp, totalSize=47.8 K 2024-11-16T09:58:10,094 DEBUG [RS:0;037c58a19ddb:37267-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 58472 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-16T09:58:10,094 DEBUG [RS:0;037c58a19ddb:37267-shortCompactions-0 {}] regionserver.HStore(1541): d9e28756f78b0555a6733ffbf6ab98bd/A is initiating minor compaction (all files) 2024-11-16T09:58:10,094 INFO [RS:0;037c58a19ddb:37267-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of d9e28756f78b0555a6733ffbf6ab98bd/A in TestAcidGuarantees,,1731751082172.d9e28756f78b0555a6733ffbf6ab98bd. 2024-11-16T09:58:10,095 INFO [RS:0;037c58a19ddb:37267-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/d9e28756f78b0555a6733ffbf6ab98bd/A/a919ab9e3a914bc59f13fe1f3f7b251e, hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/d9e28756f78b0555a6733ffbf6ab98bd/A/8900f6b9a93d4df2b2b02e97cb83983f, hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/d9e28756f78b0555a6733ffbf6ab98bd/A/5cd12febed204efabe434b45e92eccae, hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/d9e28756f78b0555a6733ffbf6ab98bd/A/0dbb7de1f967420a83f7452d01da47f7] into tmpdir=hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/d9e28756f78b0555a6733ffbf6ab98bd/.tmp, totalSize=57.1 K 2024-11-16T09:58:10,095 DEBUG [RS:0;037c58a19ddb:37267-shortCompactions-0 {}] compactions.Compactor(225): Compacting a919ab9e3a914bc59f13fe1f3f7b251e, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=137, earliestPutTs=1731751087270 2024-11-16T09:58:10,096 DEBUG [RS:0;037c58a19ddb:37267-longCompactions-0 {}] compactions.Compactor(225): Compacting 706a630aaba4416ba4ad59882a64793c, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=137, earliestPutTs=1731751087270 2024-11-16T09:58:10,096 DEBUG [RS:0;037c58a19ddb:37267-shortCompactions-0 {}] compactions.Compactor(225): Compacting 8900f6b9a93d4df2b2b02e97cb83983f, keycount=200, bloomtype=ROW, size=14.2 K, encoding=NONE, compression=NONE, seqNum=164, earliestPutTs=1731751088674 2024-11-16T09:58:10,096 DEBUG [RS:0;037c58a19ddb:37267-longCompactions-0 {}] compactions.Compactor(225): Compacting ea425dc15ba4426e82f075375750f365, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=164, earliestPutTs=1731751088674 2024-11-16T09:58:10,097 DEBUG [RS:0;037c58a19ddb:37267-shortCompactions-0 {}] compactions.Compactor(225): Compacting 5cd12febed204efabe434b45e92eccae, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=177, earliestPutTs=1731751088739 2024-11-16T09:58:10,097 DEBUG [RS:0;037c58a19ddb:37267-longCompactions-0 {}] compactions.Compactor(225): Compacting 15113ec6a9d84591914d35a99db57b4d, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=177, earliestPutTs=1731751088739 2024-11-16T09:58:10,098 DEBUG [RS:0;037c58a19ddb:37267-shortCompactions-0 {}] compactions.Compactor(225): Compacting 0dbb7de1f967420a83f7452d01da47f7, keycount=300, bloomtype=ROW, size=18.9 K, encoding=NONE, compression=NONE, seqNum=188, earliestPutTs=1731751089899 2024-11-16T09:58:10,099 DEBUG [RS:0;037c58a19ddb:37267-longCompactions-0 {}] compactions.Compactor(225): Compacting 1239686f4cbd48f49490f5231be68ce7, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=188, earliestPutTs=1731751089902 2024-11-16T09:58:10,130 INFO [RS:0;037c58a19ddb:37267-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): d9e28756f78b0555a6733ffbf6ab98bd#A#compaction#39 average throughput is 2.18 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-16T09:58:10,131 DEBUG [RS:0;037c58a19ddb:37267-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/d9e28756f78b0555a6733ffbf6ab98bd/.tmp/A/d1a50bbda2704aeead00924112cdb2ed is 50, key is test_row_0/A:col10/1731751089902/Put/seqid=0 2024-11-16T09:58:10,138 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] regionserver.HRegion(8855): Flush requested on d9e28756f78b0555a6733ffbf6ab98bd 2024-11-16T09:58:10,138 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-16T09:58:10,138 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing d9e28756f78b0555a6733ffbf6ab98bd 3/3 column families, dataSize=154.31 KB heapSize=405.05 KB 2024-11-16T09:58:10,143 INFO [RS:0;037c58a19ddb:37267-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): d9e28756f78b0555a6733ffbf6ab98bd#B#compaction#40 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-16T09:58:10,144 DEBUG [RS:0;037c58a19ddb:37267-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/d9e28756f78b0555a6733ffbf6ab98bd/.tmp/B/481575664e91483baf55d8b39ada90f2 is 50, key is test_row_0/B:col10/1731751089902/Put/seqid=0 2024-11-16T09:58:10,161 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9e28756f78b0555a6733ffbf6ab98bd, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T09:58:10,162 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39638 deadline: 1731751150156, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9e28756f78b0555a6733ffbf6ab98bd, server=037c58a19ddb,37267,1731751077844 2024-11-16T09:58:10,163 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731751082172.d9e28756f78b0555a6733ffbf6ab98bd., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 , the old value is region=TestAcidGuarantees,,1731751082172.d9e28756f78b0555a6733ffbf6ab98bd., hostname=037c58a19ddb,37267,1731751077844, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9e28756f78b0555a6733ffbf6ab98bd, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:58:10,163 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731751082172.d9e28756f78b0555a6733ffbf6ab98bd., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9e28756f78b0555a6733ffbf6ab98bd, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:58:10,163 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731751082172.d9e28756f78b0555a6733ffbf6ab98bd., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 because the exception is null or not the one we care about 2024-11-16T09:58:10,169 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9e28756f78b0555a6733ffbf6ab98bd, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T09:58:10,169 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39654 deadline: 1731751150159, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9e28756f78b0555a6733ffbf6ab98bd, server=037c58a19ddb,37267,1731751077844 2024-11-16T09:58:10,170 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731751082172.d9e28756f78b0555a6733ffbf6ab98bd., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 , the old value is region=TestAcidGuarantees,,1731751082172.d9e28756f78b0555a6733ffbf6ab98bd., hostname=037c58a19ddb,37267,1731751077844, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9e28756f78b0555a6733ffbf6ab98bd, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:58:10,170 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731751082172.d9e28756f78b0555a6733ffbf6ab98bd., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9e28756f78b0555a6733ffbf6ab98bd, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:58:10,170 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731751082172.d9e28756f78b0555a6733ffbf6ab98bd., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 because the exception is null or not the one we care about 2024-11-16T09:58:10,170 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9e28756f78b0555a6733ffbf6ab98bd, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T09:58:10,170 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9e28756f78b0555a6733ffbf6ab98bd, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T09:58:10,171 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39620 deadline: 1731751150162, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9e28756f78b0555a6733ffbf6ab98bd, server=037c58a19ddb,37267,1731751077844 2024-11-16T09:58:10,171 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39696 deadline: 1731751150160, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9e28756f78b0555a6733ffbf6ab98bd, server=037c58a19ddb,37267,1731751077844 2024-11-16T09:58:10,171 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731751082172.d9e28756f78b0555a6733ffbf6ab98bd., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 , the old value is region=TestAcidGuarantees,,1731751082172.d9e28756f78b0555a6733ffbf6ab98bd., hostname=037c58a19ddb,37267,1731751077844, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9e28756f78b0555a6733ffbf6ab98bd, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:58:10,171 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731751082172.d9e28756f78b0555a6733ffbf6ab98bd., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9e28756f78b0555a6733ffbf6ab98bd, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:58:10,171 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731751082172.d9e28756f78b0555a6733ffbf6ab98bd., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 , the old value is region=TestAcidGuarantees,,1731751082172.d9e28756f78b0555a6733ffbf6ab98bd., hostname=037c58a19ddb,37267,1731751077844, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9e28756f78b0555a6733ffbf6ab98bd, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:58:10,171 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731751082172.d9e28756f78b0555a6733ffbf6ab98bd., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 because the exception is null or not the one we care about 2024-11-16T09:58:10,171 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731751082172.d9e28756f78b0555a6733ffbf6ab98bd., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9e28756f78b0555a6733ffbf6ab98bd, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:58:10,171 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731751082172.d9e28756f78b0555a6733ffbf6ab98bd., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 because the exception is null or not the one we care about 2024-11-16T09:58:10,172 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9e28756f78b0555a6733ffbf6ab98bd, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T09:58:10,172 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37591 is added to blk_1073741876_1052 (size=12595) 2024-11-16T09:58:10,172 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39630 deadline: 1731751150162, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9e28756f78b0555a6733ffbf6ab98bd, server=037c58a19ddb,37267,1731751077844 2024-11-16T09:58:10,172 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731751082172.d9e28756f78b0555a6733ffbf6ab98bd., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 , the old value is region=TestAcidGuarantees,,1731751082172.d9e28756f78b0555a6733ffbf6ab98bd., hostname=037c58a19ddb,37267,1731751077844, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9e28756f78b0555a6733ffbf6ab98bd, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:58:10,172 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731751082172.d9e28756f78b0555a6733ffbf6ab98bd., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9e28756f78b0555a6733ffbf6ab98bd, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:58:10,173 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731751082172.d9e28756f78b0555a6733ffbf6ab98bd., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 because the exception is null or not the one we care about 2024-11-16T09:58:10,182 DEBUG [RS:0;037c58a19ddb:37267-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/d9e28756f78b0555a6733ffbf6ab98bd/.tmp/A/d1a50bbda2704aeead00924112cdb2ed as hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/d9e28756f78b0555a6733ffbf6ab98bd/A/d1a50bbda2704aeead00924112cdb2ed 2024-11-16T09:58:10,187 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/d9e28756f78b0555a6733ffbf6ab98bd/.tmp/A/b68a7ab34a8847e5a7139a4a8ec2b414 is 50, key is test_row_0/A:col10/1731751089972/Put/seqid=0 2024-11-16T09:58:10,198 INFO [RS:0;037c58a19ddb:37267-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in d9e28756f78b0555a6733ffbf6ab98bd/A of d9e28756f78b0555a6733ffbf6ab98bd into d1a50bbda2704aeead00924112cdb2ed(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-16T09:58:10,198 DEBUG [RS:0;037c58a19ddb:37267-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for d9e28756f78b0555a6733ffbf6ab98bd: 2024-11-16T09:58:10,199 INFO [RS:0;037c58a19ddb:37267-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731751082172.d9e28756f78b0555a6733ffbf6ab98bd., storeName=d9e28756f78b0555a6733ffbf6ab98bd/A, priority=12, startTime=1731751090088; duration=0sec 2024-11-16T09:58:10,199 DEBUG [RS:0;037c58a19ddb:37267-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-16T09:58:10,199 DEBUG [RS:0;037c58a19ddb:37267-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: d9e28756f78b0555a6733ffbf6ab98bd:A 2024-11-16T09:58:10,199 DEBUG [RS:0;037c58a19ddb:37267-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-16T09:58:10,201 DEBUG [RS:0;037c58a19ddb:37267-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48912 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-16T09:58:10,201 DEBUG [RS:0;037c58a19ddb:37267-shortCompactions-0 {}] regionserver.HStore(1541): d9e28756f78b0555a6733ffbf6ab98bd/C is initiating minor compaction (all files) 2024-11-16T09:58:10,201 INFO [RS:0;037c58a19ddb:37267-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of d9e28756f78b0555a6733ffbf6ab98bd/C in TestAcidGuarantees,,1731751082172.d9e28756f78b0555a6733ffbf6ab98bd. 2024-11-16T09:58:10,202 INFO [RS:0;037c58a19ddb:37267-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/d9e28756f78b0555a6733ffbf6ab98bd/C/7a3f8766b7834cda80087a8ec71ec9e7, hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/d9e28756f78b0555a6733ffbf6ab98bd/C/bc833c50bb2540ac9dda563506ca1e2a, hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/d9e28756f78b0555a6733ffbf6ab98bd/C/3ca5b8889378477bb2737c33c064ccc2, hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/d9e28756f78b0555a6733ffbf6ab98bd/C/2ae6fc50bfb0440580104fede526210d] into tmpdir=hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/d9e28756f78b0555a6733ffbf6ab98bd/.tmp, totalSize=47.8 K 2024-11-16T09:58:10,204 DEBUG [RS:0;037c58a19ddb:37267-shortCompactions-0 {}] compactions.Compactor(225): Compacting 7a3f8766b7834cda80087a8ec71ec9e7, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=137, earliestPutTs=1731751087270 2024-11-16T09:58:10,205 DEBUG [RS:0;037c58a19ddb:37267-shortCompactions-0 {}] compactions.Compactor(225): Compacting bc833c50bb2540ac9dda563506ca1e2a, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=164, earliestPutTs=1731751088674 2024-11-16T09:58:10,206 DEBUG [RS:0;037c58a19ddb:37267-shortCompactions-0 {}] compactions.Compactor(225): Compacting 3ca5b8889378477bb2737c33c064ccc2, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=177, earliestPutTs=1731751088739 2024-11-16T09:58:10,206 DEBUG [RS:0;037c58a19ddb:37267-shortCompactions-0 {}] compactions.Compactor(225): Compacting 2ae6fc50bfb0440580104fede526210d, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=188, earliestPutTs=1731751089902 2024-11-16T09:58:10,218 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37591 is added to blk_1073741877_1053 (size=12595) 2024-11-16T09:58:10,222 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37591 is added to blk_1073741878_1054 (size=12151) 2024-11-16T09:58:10,223 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=51.44 KB at sequenceid=214 (bloomFilter=true), to=hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/d9e28756f78b0555a6733ffbf6ab98bd/.tmp/A/b68a7ab34a8847e5a7139a4a8ec2b414 2024-11-16T09:58:10,234 INFO [RS:0;037c58a19ddb:37267-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): d9e28756f78b0555a6733ffbf6ab98bd#C#compaction#42 average throughput is 2.18 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-16T09:58:10,235 DEBUG [RS:0;037c58a19ddb:37267-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/d9e28756f78b0555a6733ffbf6ab98bd/.tmp/C/d6255616e9f941a9b2b01d944bbf4af9 is 50, key is test_row_0/C:col10/1731751089902/Put/seqid=0 2024-11-16T09:58:10,245 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/d9e28756f78b0555a6733ffbf6ab98bd/.tmp/B/2fbcc110db0147edba11d9d86dc65884 is 50, key is test_row_0/B:col10/1731751089972/Put/seqid=0 2024-11-16T09:58:10,266 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37591 is added to blk_1073741880_1056 (size=12151) 2024-11-16T09:58:10,268 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=51.44 KB at sequenceid=214 (bloomFilter=true), to=hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/d9e28756f78b0555a6733ffbf6ab98bd/.tmp/B/2fbcc110db0147edba11d9d86dc65884 2024-11-16T09:58:10,277 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9e28756f78b0555a6733ffbf6ab98bd, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T09:58:10,278 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39620 deadline: 1731751150274, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9e28756f78b0555a6733ffbf6ab98bd, server=037c58a19ddb,37267,1731751077844 2024-11-16T09:58:10,278 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731751082172.d9e28756f78b0555a6733ffbf6ab98bd., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 , the old value is region=TestAcidGuarantees,,1731751082172.d9e28756f78b0555a6733ffbf6ab98bd., hostname=037c58a19ddb,37267,1731751077844, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9e28756f78b0555a6733ffbf6ab98bd, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:58:10,279 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731751082172.d9e28756f78b0555a6733ffbf6ab98bd., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9e28756f78b0555a6733ffbf6ab98bd, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:58:10,279 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731751082172.d9e28756f78b0555a6733ffbf6ab98bd., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 because the exception is null or not the one we care about 2024-11-16T09:58:10,279 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9e28756f78b0555a6733ffbf6ab98bd, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T09:58:10,279 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39654 deadline: 1731751150274, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9e28756f78b0555a6733ffbf6ab98bd, server=037c58a19ddb,37267,1731751077844 2024-11-16T09:58:10,280 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37591 is added to blk_1073741879_1055 (size=12595) 2024-11-16T09:58:10,280 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731751082172.d9e28756f78b0555a6733ffbf6ab98bd., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 , the old value is region=TestAcidGuarantees,,1731751082172.d9e28756f78b0555a6733ffbf6ab98bd., hostname=037c58a19ddb,37267,1731751077844, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9e28756f78b0555a6733ffbf6ab98bd, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:58:10,280 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9e28756f78b0555a6733ffbf6ab98bd, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T09:58:10,280 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731751082172.d9e28756f78b0555a6733ffbf6ab98bd., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9e28756f78b0555a6733ffbf6ab98bd, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:58:10,280 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731751082172.d9e28756f78b0555a6733ffbf6ab98bd., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 because the exception is null or not the one we care about 2024-11-16T09:58:10,280 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39696 deadline: 1731751150274, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9e28756f78b0555a6733ffbf6ab98bd, server=037c58a19ddb,37267,1731751077844 2024-11-16T09:58:10,281 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9e28756f78b0555a6733ffbf6ab98bd, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T09:58:10,281 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731751082172.d9e28756f78b0555a6733ffbf6ab98bd., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 , the old value is region=TestAcidGuarantees,,1731751082172.d9e28756f78b0555a6733ffbf6ab98bd., hostname=037c58a19ddb,37267,1731751077844, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9e28756f78b0555a6733ffbf6ab98bd, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:58:10,281 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39638 deadline: 1731751150274, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9e28756f78b0555a6733ffbf6ab98bd, server=037c58a19ddb,37267,1731751077844 2024-11-16T09:58:10,281 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731751082172.d9e28756f78b0555a6733ffbf6ab98bd., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9e28756f78b0555a6733ffbf6ab98bd, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:58:10,281 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731751082172.d9e28756f78b0555a6733ffbf6ab98bd., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 because the exception is null or not the one we care about 2024-11-16T09:58:10,281 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731751082172.d9e28756f78b0555a6733ffbf6ab98bd., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 , the old value is region=TestAcidGuarantees,,1731751082172.d9e28756f78b0555a6733ffbf6ab98bd., hostname=037c58a19ddb,37267,1731751077844, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9e28756f78b0555a6733ffbf6ab98bd, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:58:10,282 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731751082172.d9e28756f78b0555a6733ffbf6ab98bd., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9e28756f78b0555a6733ffbf6ab98bd, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:58:10,282 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731751082172.d9e28756f78b0555a6733ffbf6ab98bd., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 because the exception is null or not the one we care about 2024-11-16T09:58:10,283 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/d9e28756f78b0555a6733ffbf6ab98bd/.tmp/C/53189d6164424f0aa851310b40b43cbb is 50, key is test_row_0/C:col10/1731751089972/Put/seqid=0 2024-11-16T09:58:10,290 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9e28756f78b0555a6733ffbf6ab98bd, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T09:58:10,290 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39630 deadline: 1731751150284, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9e28756f78b0555a6733ffbf6ab98bd, server=037c58a19ddb,37267,1731751077844 2024-11-16T09:58:10,290 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731751082172.d9e28756f78b0555a6733ffbf6ab98bd., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 , the old value is region=TestAcidGuarantees,,1731751082172.d9e28756f78b0555a6733ffbf6ab98bd., hostname=037c58a19ddb,37267,1731751077844, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9e28756f78b0555a6733ffbf6ab98bd, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:58:10,291 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731751082172.d9e28756f78b0555a6733ffbf6ab98bd., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9e28756f78b0555a6733ffbf6ab98bd, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:58:10,291 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731751082172.d9e28756f78b0555a6733ffbf6ab98bd., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 because the exception is null or not the one we care about 2024-11-16T09:58:10,293 DEBUG [RS:0;037c58a19ddb:37267-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/d9e28756f78b0555a6733ffbf6ab98bd/.tmp/C/d6255616e9f941a9b2b01d944bbf4af9 as hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/d9e28756f78b0555a6733ffbf6ab98bd/C/d6255616e9f941a9b2b01d944bbf4af9 2024-11-16T09:58:10,303 INFO [RS:0;037c58a19ddb:37267-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in d9e28756f78b0555a6733ffbf6ab98bd/C of d9e28756f78b0555a6733ffbf6ab98bd into d6255616e9f941a9b2b01d944bbf4af9(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-16T09:58:10,303 DEBUG [RS:0;037c58a19ddb:37267-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for d9e28756f78b0555a6733ffbf6ab98bd: 2024-11-16T09:58:10,303 INFO [RS:0;037c58a19ddb:37267-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731751082172.d9e28756f78b0555a6733ffbf6ab98bd., storeName=d9e28756f78b0555a6733ffbf6ab98bd/C, priority=12, startTime=1731751090088; duration=0sec 2024-11-16T09:58:10,304 DEBUG [RS:0;037c58a19ddb:37267-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-16T09:58:10,304 DEBUG [RS:0;037c58a19ddb:37267-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: d9e28756f78b0555a6733ffbf6ab98bd:C 2024-11-16T09:58:10,312 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37591 is added to blk_1073741881_1057 (size=12151) 2024-11-16T09:58:10,484 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9e28756f78b0555a6733ffbf6ab98bd, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T09:58:10,484 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39620 deadline: 1731751150483, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9e28756f78b0555a6733ffbf6ab98bd, server=037c58a19ddb,37267,1731751077844 2024-11-16T09:58:10,485 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731751082172.d9e28756f78b0555a6733ffbf6ab98bd., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 , the old value is region=TestAcidGuarantees,,1731751082172.d9e28756f78b0555a6733ffbf6ab98bd., hostname=037c58a19ddb,37267,1731751077844, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9e28756f78b0555a6733ffbf6ab98bd, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:58:10,485 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9e28756f78b0555a6733ffbf6ab98bd, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T09:58:10,485 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731751082172.d9e28756f78b0555a6733ffbf6ab98bd., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9e28756f78b0555a6733ffbf6ab98bd, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:58:10,485 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731751082172.d9e28756f78b0555a6733ffbf6ab98bd., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 because the exception is null or not the one we care about 2024-11-16T09:58:10,485 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39654 deadline: 1731751150483, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9e28756f78b0555a6733ffbf6ab98bd, server=037c58a19ddb,37267,1731751077844 2024-11-16T09:58:10,486 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731751082172.d9e28756f78b0555a6733ffbf6ab98bd., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 , the old value is region=TestAcidGuarantees,,1731751082172.d9e28756f78b0555a6733ffbf6ab98bd., hostname=037c58a19ddb,37267,1731751077844, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9e28756f78b0555a6733ffbf6ab98bd, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:58:10,486 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731751082172.d9e28756f78b0555a6733ffbf6ab98bd., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9e28756f78b0555a6733ffbf6ab98bd, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:58:10,486 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731751082172.d9e28756f78b0555a6733ffbf6ab98bd., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 because the exception is null or not the one we care about 2024-11-16T09:58:10,494 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9e28756f78b0555a6733ffbf6ab98bd, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T09:58:10,494 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39696 deadline: 1731751150493, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9e28756f78b0555a6733ffbf6ab98bd, server=037c58a19ddb,37267,1731751077844 2024-11-16T09:58:10,495 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731751082172.d9e28756f78b0555a6733ffbf6ab98bd., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 , the old value is region=TestAcidGuarantees,,1731751082172.d9e28756f78b0555a6733ffbf6ab98bd., hostname=037c58a19ddb,37267,1731751077844, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9e28756f78b0555a6733ffbf6ab98bd, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:58:10,495 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9e28756f78b0555a6733ffbf6ab98bd, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T09:58:10,495 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731751082172.d9e28756f78b0555a6733ffbf6ab98bd., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9e28756f78b0555a6733ffbf6ab98bd, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:58:10,495 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731751082172.d9e28756f78b0555a6733ffbf6ab98bd., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 because the exception is null or not the one we care about 2024-11-16T09:58:10,495 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39638 deadline: 1731751150493, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9e28756f78b0555a6733ffbf6ab98bd, server=037c58a19ddb,37267,1731751077844 2024-11-16T09:58:10,497 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731751082172.d9e28756f78b0555a6733ffbf6ab98bd., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 , the old value is region=TestAcidGuarantees,,1731751082172.d9e28756f78b0555a6733ffbf6ab98bd., hostname=037c58a19ddb,37267,1731751077844, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9e28756f78b0555a6733ffbf6ab98bd, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:58:10,497 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731751082172.d9e28756f78b0555a6733ffbf6ab98bd., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9e28756f78b0555a6733ffbf6ab98bd, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:58:10,497 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731751082172.d9e28756f78b0555a6733ffbf6ab98bd., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 because the exception is null or not the one we care about 2024-11-16T09:58:10,497 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9e28756f78b0555a6733ffbf6ab98bd, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T09:58:10,498 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39630 deadline: 1731751150493, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9e28756f78b0555a6733ffbf6ab98bd, server=037c58a19ddb,37267,1731751077844 2024-11-16T09:58:10,498 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731751082172.d9e28756f78b0555a6733ffbf6ab98bd., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 , the old value is region=TestAcidGuarantees,,1731751082172.d9e28756f78b0555a6733ffbf6ab98bd., hostname=037c58a19ddb,37267,1731751077844, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9e28756f78b0555a6733ffbf6ab98bd, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:58:10,498 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731751082172.d9e28756f78b0555a6733ffbf6ab98bd., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9e28756f78b0555a6733ffbf6ab98bd, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:58:10,498 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731751082172.d9e28756f78b0555a6733ffbf6ab98bd., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 because the exception is null or not the one we care about 2024-11-16T09:58:10,633 DEBUG [RS:0;037c58a19ddb:37267-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/d9e28756f78b0555a6733ffbf6ab98bd/.tmp/B/481575664e91483baf55d8b39ada90f2 as hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/d9e28756f78b0555a6733ffbf6ab98bd/B/481575664e91483baf55d8b39ada90f2 2024-11-16T09:58:10,642 INFO [RS:0;037c58a19ddb:37267-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in d9e28756f78b0555a6733ffbf6ab98bd/B of d9e28756f78b0555a6733ffbf6ab98bd into 481575664e91483baf55d8b39ada90f2(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-16T09:58:10,642 DEBUG [RS:0;037c58a19ddb:37267-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for d9e28756f78b0555a6733ffbf6ab98bd: 2024-11-16T09:58:10,642 INFO [RS:0;037c58a19ddb:37267-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731751082172.d9e28756f78b0555a6733ffbf6ab98bd., storeName=d9e28756f78b0555a6733ffbf6ab98bd/B, priority=12, startTime=1731751090088; duration=0sec 2024-11-16T09:58:10,642 DEBUG [RS:0;037c58a19ddb:37267-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-16T09:58:10,642 DEBUG [RS:0;037c58a19ddb:37267-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: d9e28756f78b0555a6733ffbf6ab98bd:B 2024-11-16T09:58:10,715 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=51.44 KB at sequenceid=214 (bloomFilter=true), to=hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/d9e28756f78b0555a6733ffbf6ab98bd/.tmp/C/53189d6164424f0aa851310b40b43cbb 2024-11-16T09:58:10,728 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/d9e28756f78b0555a6733ffbf6ab98bd/.tmp/A/b68a7ab34a8847e5a7139a4a8ec2b414 as hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/d9e28756f78b0555a6733ffbf6ab98bd/A/b68a7ab34a8847e5a7139a4a8ec2b414 2024-11-16T09:58:10,739 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/d9e28756f78b0555a6733ffbf6ab98bd/A/b68a7ab34a8847e5a7139a4a8ec2b414, entries=150, sequenceid=214, filesize=11.9 K 2024-11-16T09:58:10,740 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/d9e28756f78b0555a6733ffbf6ab98bd/.tmp/B/2fbcc110db0147edba11d9d86dc65884 as hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/d9e28756f78b0555a6733ffbf6ab98bd/B/2fbcc110db0147edba11d9d86dc65884 2024-11-16T09:58:10,759 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/d9e28756f78b0555a6733ffbf6ab98bd/B/2fbcc110db0147edba11d9d86dc65884, entries=150, sequenceid=214, filesize=11.9 K 2024-11-16T09:58:10,760 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/d9e28756f78b0555a6733ffbf6ab98bd/.tmp/C/53189d6164424f0aa851310b40b43cbb as hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/d9e28756f78b0555a6733ffbf6ab98bd/C/53189d6164424f0aa851310b40b43cbb 2024-11-16T09:58:10,767 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/d9e28756f78b0555a6733ffbf6ab98bd/C/53189d6164424f0aa851310b40b43cbb, entries=150, sequenceid=214, filesize=11.9 K 2024-11-16T09:58:10,772 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~154.31 KB/158010, heapSize ~405 KB/414720, currentSize=53.67 KB/54960 for d9e28756f78b0555a6733ffbf6ab98bd in 634ms, sequenceid=214, compaction requested=false 2024-11-16T09:58:10,772 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for d9e28756f78b0555a6733ffbf6ab98bd: 2024-11-16T09:58:10,783 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37877 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=9 2024-11-16T09:58:10,784 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-16T09:58:10,785 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37877 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-16T09:58:10,787 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37877 {}] procedure2.ProcedureExecutor(1139): Stored pid=11, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=11, table=TestAcidGuarantees 2024-11-16T09:58:10,788 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37877 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=11 2024-11-16T09:58:10,789 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=11, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=11, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-16T09:58:10,790 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=11, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=11, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-16T09:58:10,790 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=12, ppid=11, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-16T09:58:10,797 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] regionserver.HRegion(8855): Flush requested on d9e28756f78b0555a6733ffbf6ab98bd 2024-11-16T09:58:10,797 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-16T09:58:10,797 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing d9e28756f78b0555a6733ffbf6ab98bd 3/3 column families, dataSize=60.38 KB heapSize=158.95 KB 2024-11-16T09:58:10,810 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/d9e28756f78b0555a6733ffbf6ab98bd/.tmp/A/2108968e9c084ac5bdc90f8e037e0627 is 50, key is test_row_0/A:col10/1731751090794/Put/seqid=0 2024-11-16T09:58:10,845 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9e28756f78b0555a6733ffbf6ab98bd, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T09:58:10,845 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39696 deadline: 1731751150833, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9e28756f78b0555a6733ffbf6ab98bd, server=037c58a19ddb,37267,1731751077844 2024-11-16T09:58:10,846 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731751082172.d9e28756f78b0555a6733ffbf6ab98bd., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 , the old value is region=TestAcidGuarantees,,1731751082172.d9e28756f78b0555a6733ffbf6ab98bd., hostname=037c58a19ddb,37267,1731751077844, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9e28756f78b0555a6733ffbf6ab98bd, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:58:10,846 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37591 is added to blk_1073741882_1058 (size=14541) 2024-11-16T09:58:10,846 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731751082172.d9e28756f78b0555a6733ffbf6ab98bd., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9e28756f78b0555a6733ffbf6ab98bd, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:58:10,846 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731751082172.d9e28756f78b0555a6733ffbf6ab98bd., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 because the exception is null or not the one we care about 2024-11-16T09:58:10,847 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9e28756f78b0555a6733ffbf6ab98bd, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T09:58:10,847 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39620 deadline: 1731751150835, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9e28756f78b0555a6733ffbf6ab98bd, server=037c58a19ddb,37267,1731751077844 2024-11-16T09:58:10,847 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731751082172.d9e28756f78b0555a6733ffbf6ab98bd., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 , the old value is region=TestAcidGuarantees,,1731751082172.d9e28756f78b0555a6733ffbf6ab98bd., hostname=037c58a19ddb,37267,1731751077844, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9e28756f78b0555a6733ffbf6ab98bd, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:58:10,848 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731751082172.d9e28756f78b0555a6733ffbf6ab98bd., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9e28756f78b0555a6733ffbf6ab98bd, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:58:10,847 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9e28756f78b0555a6733ffbf6ab98bd, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T09:58:10,848 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731751082172.d9e28756f78b0555a6733ffbf6ab98bd., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 because the exception is null or not the one we care about 2024-11-16T09:58:10,848 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] ipc.CallRunner(138): callId: 71 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39654 deadline: 1731751150836, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9e28756f78b0555a6733ffbf6ab98bd, server=037c58a19ddb,37267,1731751077844 2024-11-16T09:58:10,848 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9e28756f78b0555a6733ffbf6ab98bd, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T09:58:10,848 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39638 deadline: 1731751150838, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9e28756f78b0555a6733ffbf6ab98bd, server=037c58a19ddb,37267,1731751077844 2024-11-16T09:58:10,849 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731751082172.d9e28756f78b0555a6733ffbf6ab98bd., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 , the old value is region=TestAcidGuarantees,,1731751082172.d9e28756f78b0555a6733ffbf6ab98bd., hostname=037c58a19ddb,37267,1731751077844, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9e28756f78b0555a6733ffbf6ab98bd, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:58:10,849 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731751082172.d9e28756f78b0555a6733ffbf6ab98bd., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9e28756f78b0555a6733ffbf6ab98bd, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:58:10,849 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731751082172.d9e28756f78b0555a6733ffbf6ab98bd., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 , the old value is region=TestAcidGuarantees,,1731751082172.d9e28756f78b0555a6733ffbf6ab98bd., hostname=037c58a19ddb,37267,1731751077844, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9e28756f78b0555a6733ffbf6ab98bd, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:58:10,849 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731751082172.d9e28756f78b0555a6733ffbf6ab98bd., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 because the exception is null or not the one we care about 2024-11-16T09:58:10,849 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731751082172.d9e28756f78b0555a6733ffbf6ab98bd., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9e28756f78b0555a6733ffbf6ab98bd, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:58:10,849 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731751082172.d9e28756f78b0555a6733ffbf6ab98bd., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 because the exception is null or not the one we care about 2024-11-16T09:58:10,849 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=230 (bloomFilter=true), to=hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/d9e28756f78b0555a6733ffbf6ab98bd/.tmp/A/2108968e9c084ac5bdc90f8e037e0627 2024-11-16T09:58:10,849 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9e28756f78b0555a6733ffbf6ab98bd, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T09:58:10,851 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39630 deadline: 1731751150845, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9e28756f78b0555a6733ffbf6ab98bd, server=037c58a19ddb,37267,1731751077844 2024-11-16T09:58:10,852 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731751082172.d9e28756f78b0555a6733ffbf6ab98bd., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 , the old value is region=TestAcidGuarantees,,1731751082172.d9e28756f78b0555a6733ffbf6ab98bd., hostname=037c58a19ddb,37267,1731751077844, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9e28756f78b0555a6733ffbf6ab98bd, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:58:10,852 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731751082172.d9e28756f78b0555a6733ffbf6ab98bd., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9e28756f78b0555a6733ffbf6ab98bd, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:58:10,852 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731751082172.d9e28756f78b0555a6733ffbf6ab98bd., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 because the exception is null or not the one we care about 2024-11-16T09:58:10,862 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/d9e28756f78b0555a6733ffbf6ab98bd/.tmp/B/382cc9e0996747efbc235a1970fd6c61 is 50, key is test_row_0/B:col10/1731751090794/Put/seqid=0 2024-11-16T09:58:10,873 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37591 is added to blk_1073741883_1059 (size=12151) 2024-11-16T09:58:10,893 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37877 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=11 2024-11-16T09:58:10,945 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=37267 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=12 2024-11-16T09:58:10,948 DEBUG [RS_FLUSH_OPERATIONS-regionserver/037c58a19ddb:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731751082172.d9e28756f78b0555a6733ffbf6ab98bd. 2024-11-16T09:58:10,948 DEBUG [RS_FLUSH_OPERATIONS-regionserver/037c58a19ddb:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731751082172.d9e28756f78b0555a6733ffbf6ab98bd. as already flushing 2024-11-16T09:58:10,948 DEBUG [RS_FLUSH_OPERATIONS-regionserver/037c58a19ddb:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731751082172.d9e28756f78b0555a6733ffbf6ab98bd. 2024-11-16T09:58:10,948 ERROR [RS_FLUSH_OPERATIONS-regionserver/037c58a19ddb:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] handler.RSProcedureHandler(58): pid=12 java.io.IOException: Unable to complete flush {ENCODED => d9e28756f78b0555a6733ffbf6ab98bd, NAME => 'TestAcidGuarantees,,1731751082172.d9e28756f78b0555a6733ffbf6ab98bd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T09:58:10,949 DEBUG [RS_FLUSH_OPERATIONS-regionserver/037c58a19ddb:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=12 java.io.IOException: Unable to complete flush {ENCODED => d9e28756f78b0555a6733ffbf6ab98bd, NAME => 'TestAcidGuarantees,,1731751082172.d9e28756f78b0555a6733ffbf6ab98bd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T09:58:10,950 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37877 {}] master.HMaster(4177): Remote procedure failed, pid=12 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d9e28756f78b0555a6733ffbf6ab98bd, NAME => 'TestAcidGuarantees,,1731751082172.d9e28756f78b0555a6733ffbf6ab98bd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d9e28756f78b0555a6733ffbf6ab98bd, NAME => 'TestAcidGuarantees,,1731751082172.d9e28756f78b0555a6733ffbf6ab98bd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T09:58:10,956 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9e28756f78b0555a6733ffbf6ab98bd, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T09:58:10,957 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39696 deadline: 1731751150953, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9e28756f78b0555a6733ffbf6ab98bd, server=037c58a19ddb,37267,1731751077844 2024-11-16T09:58:10,957 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9e28756f78b0555a6733ffbf6ab98bd, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T09:58:10,957 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39620 deadline: 1731751150953, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9e28756f78b0555a6733ffbf6ab98bd, server=037c58a19ddb,37267,1731751077844 2024-11-16T09:58:10,957 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731751082172.d9e28756f78b0555a6733ffbf6ab98bd., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 , the old value is region=TestAcidGuarantees,,1731751082172.d9e28756f78b0555a6733ffbf6ab98bd., hostname=037c58a19ddb,37267,1731751077844, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9e28756f78b0555a6733ffbf6ab98bd, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:58:10,957 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731751082172.d9e28756f78b0555a6733ffbf6ab98bd., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9e28756f78b0555a6733ffbf6ab98bd, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:58:10,958 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731751082172.d9e28756f78b0555a6733ffbf6ab98bd., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 because the exception is null or not the one we care about 2024-11-16T09:58:10,958 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731751082172.d9e28756f78b0555a6733ffbf6ab98bd., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 , the old value is region=TestAcidGuarantees,,1731751082172.d9e28756f78b0555a6733ffbf6ab98bd., hostname=037c58a19ddb,37267,1731751077844, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9e28756f78b0555a6733ffbf6ab98bd, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:58:10,958 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731751082172.d9e28756f78b0555a6733ffbf6ab98bd., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9e28756f78b0555a6733ffbf6ab98bd, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:58:10,958 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731751082172.d9e28756f78b0555a6733ffbf6ab98bd., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 because the exception is null or not the one we care about 2024-11-16T09:58:10,957 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9e28756f78b0555a6733ffbf6ab98bd, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T09:58:10,966 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] ipc.CallRunner(138): callId: 72 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39654 deadline: 1731751150953, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9e28756f78b0555a6733ffbf6ab98bd, server=037c58a19ddb,37267,1731751077844 2024-11-16T09:58:10,967 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731751082172.d9e28756f78b0555a6733ffbf6ab98bd., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 , the old value is region=TestAcidGuarantees,,1731751082172.d9e28756f78b0555a6733ffbf6ab98bd., hostname=037c58a19ddb,37267,1731751077844, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9e28756f78b0555a6733ffbf6ab98bd, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:58:10,967 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731751082172.d9e28756f78b0555a6733ffbf6ab98bd., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9e28756f78b0555a6733ffbf6ab98bd, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:58:10,967 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731751082172.d9e28756f78b0555a6733ffbf6ab98bd., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 because the exception is null or not the one we care about 2024-11-16T09:58:10,972 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9e28756f78b0555a6733ffbf6ab98bd, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T09:58:10,973 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39638 deadline: 1731751150964, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9e28756f78b0555a6733ffbf6ab98bd, server=037c58a19ddb,37267,1731751077844 2024-11-16T09:58:10,973 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731751082172.d9e28756f78b0555a6733ffbf6ab98bd., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 , the old value is region=TestAcidGuarantees,,1731751082172.d9e28756f78b0555a6733ffbf6ab98bd., hostname=037c58a19ddb,37267,1731751077844, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9e28756f78b0555a6733ffbf6ab98bd, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:58:10,973 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731751082172.d9e28756f78b0555a6733ffbf6ab98bd., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9e28756f78b0555a6733ffbf6ab98bd, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:58:10,973 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731751082172.d9e28756f78b0555a6733ffbf6ab98bd., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 because the exception is null or not the one we care about 2024-11-16T09:58:10,973 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9e28756f78b0555a6733ffbf6ab98bd, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T09:58:10,974 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39630 deadline: 1731751150967, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9e28756f78b0555a6733ffbf6ab98bd, server=037c58a19ddb,37267,1731751077844 2024-11-16T09:58:10,974 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731751082172.d9e28756f78b0555a6733ffbf6ab98bd., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 , the old value is region=TestAcidGuarantees,,1731751082172.d9e28756f78b0555a6733ffbf6ab98bd., hostname=037c58a19ddb,37267,1731751077844, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9e28756f78b0555a6733ffbf6ab98bd, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:58:10,974 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731751082172.d9e28756f78b0555a6733ffbf6ab98bd., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9e28756f78b0555a6733ffbf6ab98bd, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:58:10,974 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731751082172.d9e28756f78b0555a6733ffbf6ab98bd., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 because the exception is null or not the one we care about 2024-11-16T09:58:11,103 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37877 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=11 2024-11-16T09:58:11,107 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=37267 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=12 2024-11-16T09:58:11,107 DEBUG [RS_FLUSH_OPERATIONS-regionserver/037c58a19ddb:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731751082172.d9e28756f78b0555a6733ffbf6ab98bd. 2024-11-16T09:58:11,107 DEBUG [RS_FLUSH_OPERATIONS-regionserver/037c58a19ddb:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731751082172.d9e28756f78b0555a6733ffbf6ab98bd. as already flushing 2024-11-16T09:58:11,107 DEBUG [RS_FLUSH_OPERATIONS-regionserver/037c58a19ddb:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731751082172.d9e28756f78b0555a6733ffbf6ab98bd. 2024-11-16T09:58:11,107 ERROR [RS_FLUSH_OPERATIONS-regionserver/037c58a19ddb:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] handler.RSProcedureHandler(58): pid=12 java.io.IOException: Unable to complete flush {ENCODED => d9e28756f78b0555a6733ffbf6ab98bd, NAME => 'TestAcidGuarantees,,1731751082172.d9e28756f78b0555a6733ffbf6ab98bd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T09:58:11,107 DEBUG [RS_FLUSH_OPERATIONS-regionserver/037c58a19ddb:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=12 java.io.IOException: Unable to complete flush {ENCODED => d9e28756f78b0555a6733ffbf6ab98bd, NAME => 'TestAcidGuarantees,,1731751082172.d9e28756f78b0555a6733ffbf6ab98bd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T09:58:11,108 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37877 {}] master.HMaster(4177): Remote procedure failed, pid=12 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d9e28756f78b0555a6733ffbf6ab98bd, NAME => 'TestAcidGuarantees,,1731751082172.d9e28756f78b0555a6733ffbf6ab98bd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d9e28756f78b0555a6733ffbf6ab98bd, NAME => 'TestAcidGuarantees,,1731751082172.d9e28756f78b0555a6733ffbf6ab98bd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T09:58:11,168 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9e28756f78b0555a6733ffbf6ab98bd, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T09:58:11,168 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39620 deadline: 1731751151164, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9e28756f78b0555a6733ffbf6ab98bd, server=037c58a19ddb,37267,1731751077844 2024-11-16T09:58:11,169 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9e28756f78b0555a6733ffbf6ab98bd, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T09:58:11,169 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39696 deadline: 1731751151164, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9e28756f78b0555a6733ffbf6ab98bd, server=037c58a19ddb,37267,1731751077844 2024-11-16T09:58:11,169 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731751082172.d9e28756f78b0555a6733ffbf6ab98bd., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 , the old value is region=TestAcidGuarantees,,1731751082172.d9e28756f78b0555a6733ffbf6ab98bd., hostname=037c58a19ddb,37267,1731751077844, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9e28756f78b0555a6733ffbf6ab98bd, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:58:11,169 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731751082172.d9e28756f78b0555a6733ffbf6ab98bd., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9e28756f78b0555a6733ffbf6ab98bd, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:58:11,169 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731751082172.d9e28756f78b0555a6733ffbf6ab98bd., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 because the exception is null or not the one we care about 2024-11-16T09:58:11,169 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731751082172.d9e28756f78b0555a6733ffbf6ab98bd., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 , the old value is region=TestAcidGuarantees,,1731751082172.d9e28756f78b0555a6733ffbf6ab98bd., hostname=037c58a19ddb,37267,1731751077844, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9e28756f78b0555a6733ffbf6ab98bd, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:58:11,169 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731751082172.d9e28756f78b0555a6733ffbf6ab98bd., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9e28756f78b0555a6733ffbf6ab98bd, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:58:11,169 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731751082172.d9e28756f78b0555a6733ffbf6ab98bd., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 because the exception is null or not the one we care about 2024-11-16T09:58:11,173 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9e28756f78b0555a6733ffbf6ab98bd, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T09:58:11,174 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] ipc.CallRunner(138): callId: 73 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39654 deadline: 1731751151173, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9e28756f78b0555a6733ffbf6ab98bd, server=037c58a19ddb,37267,1731751077844 2024-11-16T09:58:11,174 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731751082172.d9e28756f78b0555a6733ffbf6ab98bd., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 , the old value is region=TestAcidGuarantees,,1731751082172.d9e28756f78b0555a6733ffbf6ab98bd., hostname=037c58a19ddb,37267,1731751077844, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9e28756f78b0555a6733ffbf6ab98bd, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:58:11,174 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731751082172.d9e28756f78b0555a6733ffbf6ab98bd., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9e28756f78b0555a6733ffbf6ab98bd, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:58:11,174 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731751082172.d9e28756f78b0555a6733ffbf6ab98bd., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 because the exception is null or not the one we care about 2024-11-16T09:58:11,184 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9e28756f78b0555a6733ffbf6ab98bd, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T09:58:11,185 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39638 deadline: 1731751151182, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9e28756f78b0555a6733ffbf6ab98bd, server=037c58a19ddb,37267,1731751077844 2024-11-16T09:58:11,185 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731751082172.d9e28756f78b0555a6733ffbf6ab98bd., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 , the old value is region=TestAcidGuarantees,,1731751082172.d9e28756f78b0555a6733ffbf6ab98bd., hostname=037c58a19ddb,37267,1731751077844, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9e28756f78b0555a6733ffbf6ab98bd, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:58:11,185 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731751082172.d9e28756f78b0555a6733ffbf6ab98bd., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9e28756f78b0555a6733ffbf6ab98bd, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:58:11,185 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9e28756f78b0555a6733ffbf6ab98bd, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T09:58:11,185 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731751082172.d9e28756f78b0555a6733ffbf6ab98bd., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 because the exception is null or not the one we care about 2024-11-16T09:58:11,185 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39630 deadline: 1731751151182, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9e28756f78b0555a6733ffbf6ab98bd, server=037c58a19ddb,37267,1731751077844 2024-11-16T09:58:11,186 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731751082172.d9e28756f78b0555a6733ffbf6ab98bd., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 , the old value is region=TestAcidGuarantees,,1731751082172.d9e28756f78b0555a6733ffbf6ab98bd., hostname=037c58a19ddb,37267,1731751077844, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9e28756f78b0555a6733ffbf6ab98bd, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:58:11,186 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731751082172.d9e28756f78b0555a6733ffbf6ab98bd., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9e28756f78b0555a6733ffbf6ab98bd, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:58:11,186 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731751082172.d9e28756f78b0555a6733ffbf6ab98bd., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 because the exception is null or not the one we care about 2024-11-16T09:58:11,261 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=37267 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=12 2024-11-16T09:58:11,261 DEBUG [RS_FLUSH_OPERATIONS-regionserver/037c58a19ddb:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731751082172.d9e28756f78b0555a6733ffbf6ab98bd. 2024-11-16T09:58:11,262 DEBUG [RS_FLUSH_OPERATIONS-regionserver/037c58a19ddb:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731751082172.d9e28756f78b0555a6733ffbf6ab98bd. as already flushing 2024-11-16T09:58:11,262 DEBUG [RS_FLUSH_OPERATIONS-regionserver/037c58a19ddb:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731751082172.d9e28756f78b0555a6733ffbf6ab98bd. 2024-11-16T09:58:11,262 ERROR [RS_FLUSH_OPERATIONS-regionserver/037c58a19ddb:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] handler.RSProcedureHandler(58): pid=12 java.io.IOException: Unable to complete flush {ENCODED => d9e28756f78b0555a6733ffbf6ab98bd, NAME => 'TestAcidGuarantees,,1731751082172.d9e28756f78b0555a6733ffbf6ab98bd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T09:58:11,262 DEBUG [RS_FLUSH_OPERATIONS-regionserver/037c58a19ddb:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=12 java.io.IOException: Unable to complete flush {ENCODED => d9e28756f78b0555a6733ffbf6ab98bd, NAME => 'TestAcidGuarantees,,1731751082172.d9e28756f78b0555a6733ffbf6ab98bd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T09:58:11,263 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37877 {}] master.HMaster(4177): Remote procedure failed, pid=12 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d9e28756f78b0555a6733ffbf6ab98bd, NAME => 'TestAcidGuarantees,,1731751082172.d9e28756f78b0555a6733ffbf6ab98bd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d9e28756f78b0555a6733ffbf6ab98bd, NAME => 'TestAcidGuarantees,,1731751082172.d9e28756f78b0555a6733ffbf6ab98bd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T09:58:11,274 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=230 (bloomFilter=true), to=hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/d9e28756f78b0555a6733ffbf6ab98bd/.tmp/B/382cc9e0996747efbc235a1970fd6c61 2024-11-16T09:58:11,285 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/d9e28756f78b0555a6733ffbf6ab98bd/.tmp/C/525e25af8af54a7a8646f1ffeb98e1d5 is 50, key is test_row_0/C:col10/1731751090794/Put/seqid=0 2024-11-16T09:58:11,307 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37591 is added to blk_1073741884_1060 (size=12151) 2024-11-16T09:58:11,308 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=230 (bloomFilter=true), to=hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/d9e28756f78b0555a6733ffbf6ab98bd/.tmp/C/525e25af8af54a7a8646f1ffeb98e1d5 2024-11-16T09:58:11,316 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/d9e28756f78b0555a6733ffbf6ab98bd/.tmp/A/2108968e9c084ac5bdc90f8e037e0627 as hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/d9e28756f78b0555a6733ffbf6ab98bd/A/2108968e9c084ac5bdc90f8e037e0627 2024-11-16T09:58:11,323 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/d9e28756f78b0555a6733ffbf6ab98bd/A/2108968e9c084ac5bdc90f8e037e0627, entries=200, sequenceid=230, filesize=14.2 K 2024-11-16T09:58:11,326 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/d9e28756f78b0555a6733ffbf6ab98bd/.tmp/B/382cc9e0996747efbc235a1970fd6c61 as hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/d9e28756f78b0555a6733ffbf6ab98bd/B/382cc9e0996747efbc235a1970fd6c61 2024-11-16T09:58:11,335 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/d9e28756f78b0555a6733ffbf6ab98bd/B/382cc9e0996747efbc235a1970fd6c61, entries=150, sequenceid=230, filesize=11.9 K 2024-11-16T09:58:11,336 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/d9e28756f78b0555a6733ffbf6ab98bd/.tmp/C/525e25af8af54a7a8646f1ffeb98e1d5 as hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/d9e28756f78b0555a6733ffbf6ab98bd/C/525e25af8af54a7a8646f1ffeb98e1d5 2024-11-16T09:58:11,343 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/d9e28756f78b0555a6733ffbf6ab98bd/C/525e25af8af54a7a8646f1ffeb98e1d5, entries=150, sequenceid=230, filesize=11.9 K 2024-11-16T09:58:11,345 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~67.09 KB/68700, heapSize ~176.48 KB/180720, currentSize=134.18 KB/137400 for d9e28756f78b0555a6733ffbf6ab98bd in 547ms, sequenceid=230, compaction requested=true 2024-11-16T09:58:11,345 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for d9e28756f78b0555a6733ffbf6ab98bd: 2024-11-16T09:58:11,345 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store d9e28756f78b0555a6733ffbf6ab98bd:A, priority=-2147483648, current under compaction store size is 1 2024-11-16T09:58:11,345 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-16T09:58:11,345 DEBUG [RS:0;037c58a19ddb:37267-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-16T09:58:11,345 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store d9e28756f78b0555a6733ffbf6ab98bd:B, priority=-2147483648, current under compaction store size is 2 2024-11-16T09:58:11,345 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-16T09:58:11,345 DEBUG [RS:0;037c58a19ddb:37267-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-16T09:58:11,345 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store d9e28756f78b0555a6733ffbf6ab98bd:C, priority=-2147483648, current under compaction store size is 3 2024-11-16T09:58:11,345 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-16T09:58:11,347 DEBUG [RS:0;037c58a19ddb:37267-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 39287 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-16T09:58:11,347 DEBUG [RS:0;037c58a19ddb:37267-shortCompactions-0 {}] regionserver.HStore(1541): d9e28756f78b0555a6733ffbf6ab98bd/A is initiating minor compaction (all files) 2024-11-16T09:58:11,347 INFO [RS:0;037c58a19ddb:37267-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of d9e28756f78b0555a6733ffbf6ab98bd/A in TestAcidGuarantees,,1731751082172.d9e28756f78b0555a6733ffbf6ab98bd. 2024-11-16T09:58:11,347 INFO [RS:0;037c58a19ddb:37267-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/d9e28756f78b0555a6733ffbf6ab98bd/A/d1a50bbda2704aeead00924112cdb2ed, hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/d9e28756f78b0555a6733ffbf6ab98bd/A/b68a7ab34a8847e5a7139a4a8ec2b414, hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/d9e28756f78b0555a6733ffbf6ab98bd/A/2108968e9c084ac5bdc90f8e037e0627] into tmpdir=hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/d9e28756f78b0555a6733ffbf6ab98bd/.tmp, totalSize=38.4 K 2024-11-16T09:58:11,348 DEBUG [RS:0;037c58a19ddb:37267-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36897 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-16T09:58:11,348 DEBUG [RS:0;037c58a19ddb:37267-longCompactions-0 {}] regionserver.HStore(1541): d9e28756f78b0555a6733ffbf6ab98bd/B is initiating minor compaction (all files) 2024-11-16T09:58:11,348 INFO [RS:0;037c58a19ddb:37267-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of d9e28756f78b0555a6733ffbf6ab98bd/B in TestAcidGuarantees,,1731751082172.d9e28756f78b0555a6733ffbf6ab98bd. 2024-11-16T09:58:11,348 INFO [RS:0;037c58a19ddb:37267-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/d9e28756f78b0555a6733ffbf6ab98bd/B/481575664e91483baf55d8b39ada90f2, hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/d9e28756f78b0555a6733ffbf6ab98bd/B/2fbcc110db0147edba11d9d86dc65884, hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/d9e28756f78b0555a6733ffbf6ab98bd/B/382cc9e0996747efbc235a1970fd6c61] into tmpdir=hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/d9e28756f78b0555a6733ffbf6ab98bd/.tmp, totalSize=36.0 K 2024-11-16T09:58:11,349 DEBUG [RS:0;037c58a19ddb:37267-shortCompactions-0 {}] compactions.Compactor(225): Compacting d1a50bbda2704aeead00924112cdb2ed, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=188, earliestPutTs=1731751089902 2024-11-16T09:58:11,349 DEBUG [RS:0;037c58a19ddb:37267-longCompactions-0 {}] compactions.Compactor(225): Compacting 481575664e91483baf55d8b39ada90f2, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=188, earliestPutTs=1731751089902 2024-11-16T09:58:11,350 DEBUG [RS:0;037c58a19ddb:37267-shortCompactions-0 {}] compactions.Compactor(225): Compacting b68a7ab34a8847e5a7139a4a8ec2b414, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=214, earliestPutTs=1731751089972 2024-11-16T09:58:11,350 DEBUG [RS:0;037c58a19ddb:37267-longCompactions-0 {}] compactions.Compactor(225): Compacting 2fbcc110db0147edba11d9d86dc65884, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=214, earliestPutTs=1731751089972 2024-11-16T09:58:11,351 DEBUG [RS:0;037c58a19ddb:37267-shortCompactions-0 {}] compactions.Compactor(225): Compacting 2108968e9c084ac5bdc90f8e037e0627, keycount=200, bloomtype=ROW, size=14.2 K, encoding=NONE, compression=NONE, seqNum=230, earliestPutTs=1731751090146 2024-11-16T09:58:11,351 DEBUG [RS:0;037c58a19ddb:37267-longCompactions-0 {}] compactions.Compactor(225): Compacting 382cc9e0996747efbc235a1970fd6c61, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=230, earliestPutTs=1731751090158 2024-11-16T09:58:11,368 INFO [RS:0;037c58a19ddb:37267-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): d9e28756f78b0555a6733ffbf6ab98bd#A#compaction#48 average throughput is 1.64 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-16T09:58:11,369 DEBUG [RS:0;037c58a19ddb:37267-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/d9e28756f78b0555a6733ffbf6ab98bd/.tmp/A/f18628f693944e64aadc099e58410220 is 50, key is test_row_0/A:col10/1731751090794/Put/seqid=0 2024-11-16T09:58:11,382 INFO [RS:0;037c58a19ddb:37267-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): d9e28756f78b0555a6733ffbf6ab98bd#B#compaction#49 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-16T09:58:11,383 DEBUG [RS:0;037c58a19ddb:37267-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/d9e28756f78b0555a6733ffbf6ab98bd/.tmp/B/a93e2c2d05e94cf0b36912ca7912192a is 50, key is test_row_0/B:col10/1731751090794/Put/seqid=0 2024-11-16T09:58:11,392 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37591 is added to blk_1073741885_1061 (size=12697) 2024-11-16T09:58:11,408 DEBUG [RS:0;037c58a19ddb:37267-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/d9e28756f78b0555a6733ffbf6ab98bd/.tmp/A/f18628f693944e64aadc099e58410220 as hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/d9e28756f78b0555a6733ffbf6ab98bd/A/f18628f693944e64aadc099e58410220 2024-11-16T09:58:11,413 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37877 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=11 2024-11-16T09:58:11,416 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=37267 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=12 2024-11-16T09:58:11,417 DEBUG [RS_FLUSH_OPERATIONS-regionserver/037c58a19ddb:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731751082172.d9e28756f78b0555a6733ffbf6ab98bd. 2024-11-16T09:58:11,417 INFO [RS_FLUSH_OPERATIONS-regionserver/037c58a19ddb:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegion(2902): Flushing d9e28756f78b0555a6733ffbf6ab98bd 3/3 column families, dataSize=134.18 KB heapSize=352.31 KB 2024-11-16T09:58:11,419 INFO [RS:0;037c58a19ddb:37267-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in d9e28756f78b0555a6733ffbf6ab98bd/A of d9e28756f78b0555a6733ffbf6ab98bd into f18628f693944e64aadc099e58410220(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-16T09:58:11,419 DEBUG [RS:0;037c58a19ddb:37267-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for d9e28756f78b0555a6733ffbf6ab98bd: 2024-11-16T09:58:11,419 INFO [RS:0;037c58a19ddb:37267-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731751082172.d9e28756f78b0555a6733ffbf6ab98bd., storeName=d9e28756f78b0555a6733ffbf6ab98bd/A, priority=13, startTime=1731751091345; duration=0sec 2024-11-16T09:58:11,419 DEBUG [RS:0;037c58a19ddb:37267-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-16T09:58:11,419 DEBUG [RS:0;037c58a19ddb:37267-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: d9e28756f78b0555a6733ffbf6ab98bd:A 2024-11-16T09:58:11,419 DEBUG [RS:0;037c58a19ddb:37267-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-16T09:58:11,421 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37591 is added to blk_1073741886_1062 (size=12697) 2024-11-16T09:58:11,422 DEBUG [RS:0;037c58a19ddb:37267-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36897 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-16T09:58:11,422 DEBUG [RS:0;037c58a19ddb:37267-shortCompactions-0 {}] regionserver.HStore(1541): d9e28756f78b0555a6733ffbf6ab98bd/C is initiating minor compaction (all files) 2024-11-16T09:58:11,422 INFO [RS:0;037c58a19ddb:37267-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of d9e28756f78b0555a6733ffbf6ab98bd/C in TestAcidGuarantees,,1731751082172.d9e28756f78b0555a6733ffbf6ab98bd. 2024-11-16T09:58:11,422 INFO [RS:0;037c58a19ddb:37267-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/d9e28756f78b0555a6733ffbf6ab98bd/C/d6255616e9f941a9b2b01d944bbf4af9, hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/d9e28756f78b0555a6733ffbf6ab98bd/C/53189d6164424f0aa851310b40b43cbb, hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/d9e28756f78b0555a6733ffbf6ab98bd/C/525e25af8af54a7a8646f1ffeb98e1d5] into tmpdir=hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/d9e28756f78b0555a6733ffbf6ab98bd/.tmp, totalSize=36.0 K 2024-11-16T09:58:11,423 DEBUG [RS:0;037c58a19ddb:37267-shortCompactions-0 {}] compactions.Compactor(225): Compacting d6255616e9f941a9b2b01d944bbf4af9, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=188, earliestPutTs=1731751089902 2024-11-16T09:58:11,424 DEBUG [RS:0;037c58a19ddb:37267-shortCompactions-0 {}] compactions.Compactor(225): Compacting 53189d6164424f0aa851310b40b43cbb, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=214, earliestPutTs=1731751089972 2024-11-16T09:58:11,424 DEBUG [RS:0;037c58a19ddb:37267-shortCompactions-0 {}] compactions.Compactor(225): Compacting 525e25af8af54a7a8646f1ffeb98e1d5, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=230, earliestPutTs=1731751090158 2024-11-16T09:58:11,426 DEBUG [RS_FLUSH_OPERATIONS-regionserver/037c58a19ddb:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/d9e28756f78b0555a6733ffbf6ab98bd/.tmp/A/24edc08c1cf04c16bf00fef927979300 is 50, key is test_row_0/A:col10/1731751090836/Put/seqid=0 2024-11-16T09:58:11,448 INFO [RS:0;037c58a19ddb:37267-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): d9e28756f78b0555a6733ffbf6ab98bd#C#compaction#51 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-16T09:58:11,449 DEBUG [RS:0;037c58a19ddb:37267-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/d9e28756f78b0555a6733ffbf6ab98bd/.tmp/C/79c45b5c5f474c3ea9417e42d3647139 is 50, key is test_row_0/C:col10/1731751090794/Put/seqid=0 2024-11-16T09:58:11,458 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37591 is added to blk_1073741887_1063 (size=12151) 2024-11-16T09:58:11,459 INFO [RS_FLUSH_OPERATIONS-regionserver/037c58a19ddb:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=254 (bloomFilter=true), to=hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/d9e28756f78b0555a6733ffbf6ab98bd/.tmp/A/24edc08c1cf04c16bf00fef927979300 2024-11-16T09:58:11,475 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731751082172.d9e28756f78b0555a6733ffbf6ab98bd. as already flushing 2024-11-16T09:58:11,475 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] regionserver.HRegion(8855): Flush requested on d9e28756f78b0555a6733ffbf6ab98bd 2024-11-16T09:58:11,476 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37591 is added to blk_1073741888_1064 (size=12697) 2024-11-16T09:58:11,477 DEBUG [RS_FLUSH_OPERATIONS-regionserver/037c58a19ddb:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/d9e28756f78b0555a6733ffbf6ab98bd/.tmp/B/836a5918e6164bf5a0d6607cf30b4271 is 50, key is test_row_0/B:col10/1731751090836/Put/seqid=0 2024-11-16T09:58:11,493 DEBUG [RS:0;037c58a19ddb:37267-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/d9e28756f78b0555a6733ffbf6ab98bd/.tmp/C/79c45b5c5f474c3ea9417e42d3647139 as hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/d9e28756f78b0555a6733ffbf6ab98bd/C/79c45b5c5f474c3ea9417e42d3647139 2024-11-16T09:58:11,505 INFO [RS:0;037c58a19ddb:37267-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in d9e28756f78b0555a6733ffbf6ab98bd/C of d9e28756f78b0555a6733ffbf6ab98bd into 79c45b5c5f474c3ea9417e42d3647139(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-16T09:58:11,505 DEBUG [RS:0;037c58a19ddb:37267-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for d9e28756f78b0555a6733ffbf6ab98bd: 2024-11-16T09:58:11,505 INFO [RS:0;037c58a19ddb:37267-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731751082172.d9e28756f78b0555a6733ffbf6ab98bd., storeName=d9e28756f78b0555a6733ffbf6ab98bd/C, priority=13, startTime=1731751091345; duration=0sec 2024-11-16T09:58:11,505 DEBUG [RS:0;037c58a19ddb:37267-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-16T09:58:11,505 DEBUG [RS:0;037c58a19ddb:37267-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: d9e28756f78b0555a6733ffbf6ab98bd:C 2024-11-16T09:58:11,507 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9e28756f78b0555a6733ffbf6ab98bd, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T09:58:11,508 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] ipc.CallRunner(138): callId: 71 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39696 deadline: 1731751151497, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9e28756f78b0555a6733ffbf6ab98bd, server=037c58a19ddb,37267,1731751077844 2024-11-16T09:58:11,508 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9e28756f78b0555a6733ffbf6ab98bd, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T09:58:11,509 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] ipc.CallRunner(138): callId: 75 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39620 deadline: 1731751151498, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9e28756f78b0555a6733ffbf6ab98bd, server=037c58a19ddb,37267,1731751077844 2024-11-16T09:58:11,509 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731751082172.d9e28756f78b0555a6733ffbf6ab98bd., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 , the old value is region=TestAcidGuarantees,,1731751082172.d9e28756f78b0555a6733ffbf6ab98bd., hostname=037c58a19ddb,37267,1731751077844, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9e28756f78b0555a6733ffbf6ab98bd, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:58:11,509 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731751082172.d9e28756f78b0555a6733ffbf6ab98bd., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9e28756f78b0555a6733ffbf6ab98bd, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:58:11,509 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731751082172.d9e28756f78b0555a6733ffbf6ab98bd., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 because the exception is null or not the one we care about 2024-11-16T09:58:11,509 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37591 is added to blk_1073741889_1065 (size=12151) 2024-11-16T09:58:11,509 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9e28756f78b0555a6733ffbf6ab98bd, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T09:58:11,509 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731751082172.d9e28756f78b0555a6733ffbf6ab98bd., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 , the old value is region=TestAcidGuarantees,,1731751082172.d9e28756f78b0555a6733ffbf6ab98bd., hostname=037c58a19ddb,37267,1731751077844, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9e28756f78b0555a6733ffbf6ab98bd, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:58:11,509 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39638 deadline: 1731751151501, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9e28756f78b0555a6733ffbf6ab98bd, server=037c58a19ddb,37267,1731751077844 2024-11-16T09:58:11,510 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731751082172.d9e28756f78b0555a6733ffbf6ab98bd., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9e28756f78b0555a6733ffbf6ab98bd, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:58:11,510 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731751082172.d9e28756f78b0555a6733ffbf6ab98bd., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 because the exception is null or not the one we care about 2024-11-16T09:58:11,510 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731751082172.d9e28756f78b0555a6733ffbf6ab98bd., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 , the old value is region=TestAcidGuarantees,,1731751082172.d9e28756f78b0555a6733ffbf6ab98bd., hostname=037c58a19ddb,37267,1731751077844, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9e28756f78b0555a6733ffbf6ab98bd, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:58:11,510 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9e28756f78b0555a6733ffbf6ab98bd, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T09:58:11,510 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731751082172.d9e28756f78b0555a6733ffbf6ab98bd., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9e28756f78b0555a6733ffbf6ab98bd, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:58:11,510 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731751082172.d9e28756f78b0555a6733ffbf6ab98bd., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 because the exception is null or not the one we care about 2024-11-16T09:58:11,510 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] ipc.CallRunner(138): callId: 76 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39654 deadline: 1731751151503, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9e28756f78b0555a6733ffbf6ab98bd, server=037c58a19ddb,37267,1731751077844 2024-11-16T09:58:11,510 INFO [RS_FLUSH_OPERATIONS-regionserver/037c58a19ddb:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=254 (bloomFilter=true), to=hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/d9e28756f78b0555a6733ffbf6ab98bd/.tmp/B/836a5918e6164bf5a0d6607cf30b4271 2024-11-16T09:58:11,511 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731751082172.d9e28756f78b0555a6733ffbf6ab98bd., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 , the old value is region=TestAcidGuarantees,,1731751082172.d9e28756f78b0555a6733ffbf6ab98bd., hostname=037c58a19ddb,37267,1731751077844, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9e28756f78b0555a6733ffbf6ab98bd, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:58:11,511 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9e28756f78b0555a6733ffbf6ab98bd, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T09:58:11,511 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731751082172.d9e28756f78b0555a6733ffbf6ab98bd., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9e28756f78b0555a6733ffbf6ab98bd, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:58:11,511 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731751082172.d9e28756f78b0555a6733ffbf6ab98bd., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 because the exception is null or not the one we care about 2024-11-16T09:58:11,511 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39630 deadline: 1731751151504, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9e28756f78b0555a6733ffbf6ab98bd, server=037c58a19ddb,37267,1731751077844 2024-11-16T09:58:11,511 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731751082172.d9e28756f78b0555a6733ffbf6ab98bd., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 , the old value is region=TestAcidGuarantees,,1731751082172.d9e28756f78b0555a6733ffbf6ab98bd., hostname=037c58a19ddb,37267,1731751077844, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9e28756f78b0555a6733ffbf6ab98bd, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:58:11,511 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731751082172.d9e28756f78b0555a6733ffbf6ab98bd., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9e28756f78b0555a6733ffbf6ab98bd, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:58:11,511 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731751082172.d9e28756f78b0555a6733ffbf6ab98bd., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 because the exception is null or not the one we care about 2024-11-16T09:58:11,522 DEBUG [RS_FLUSH_OPERATIONS-regionserver/037c58a19ddb:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/d9e28756f78b0555a6733ffbf6ab98bd/.tmp/C/c8dd3aac45dc4fd3a068c59808acb2c5 is 50, key is test_row_0/C:col10/1731751090836/Put/seqid=0 2024-11-16T09:58:11,542 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37591 is added to blk_1073741890_1066 (size=12151) 2024-11-16T09:58:11,543 INFO [RS_FLUSH_OPERATIONS-regionserver/037c58a19ddb:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=254 (bloomFilter=true), to=hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/d9e28756f78b0555a6733ffbf6ab98bd/.tmp/C/c8dd3aac45dc4fd3a068c59808acb2c5 2024-11-16T09:58:11,564 DEBUG [RS_FLUSH_OPERATIONS-regionserver/037c58a19ddb:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/d9e28756f78b0555a6733ffbf6ab98bd/.tmp/A/24edc08c1cf04c16bf00fef927979300 as hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/d9e28756f78b0555a6733ffbf6ab98bd/A/24edc08c1cf04c16bf00fef927979300 2024-11-16T09:58:11,573 INFO [RS_FLUSH_OPERATIONS-regionserver/037c58a19ddb:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/d9e28756f78b0555a6733ffbf6ab98bd/A/24edc08c1cf04c16bf00fef927979300, entries=150, sequenceid=254, filesize=11.9 K 2024-11-16T09:58:11,575 DEBUG [RS_FLUSH_OPERATIONS-regionserver/037c58a19ddb:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/d9e28756f78b0555a6733ffbf6ab98bd/.tmp/B/836a5918e6164bf5a0d6607cf30b4271 as hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/d9e28756f78b0555a6733ffbf6ab98bd/B/836a5918e6164bf5a0d6607cf30b4271 2024-11-16T09:58:11,583 INFO [RS_FLUSH_OPERATIONS-regionserver/037c58a19ddb:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/d9e28756f78b0555a6733ffbf6ab98bd/B/836a5918e6164bf5a0d6607cf30b4271, entries=150, sequenceid=254, filesize=11.9 K 2024-11-16T09:58:11,584 DEBUG [RS_FLUSH_OPERATIONS-regionserver/037c58a19ddb:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/d9e28756f78b0555a6733ffbf6ab98bd/.tmp/C/c8dd3aac45dc4fd3a068c59808acb2c5 as hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/d9e28756f78b0555a6733ffbf6ab98bd/C/c8dd3aac45dc4fd3a068c59808acb2c5 2024-11-16T09:58:11,592 INFO [RS_FLUSH_OPERATIONS-regionserver/037c58a19ddb:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/d9e28756f78b0555a6733ffbf6ab98bd/C/c8dd3aac45dc4fd3a068c59808acb2c5, entries=150, sequenceid=254, filesize=11.9 K 2024-11-16T09:58:11,594 INFO [RS_FLUSH_OPERATIONS-regionserver/037c58a19ddb:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegion(3140): Finished flush of dataSize ~134.18 KB/137400, heapSize ~352.27 KB/360720, currentSize=67.09 KB/68700 for d9e28756f78b0555a6733ffbf6ab98bd in 177ms, sequenceid=254, compaction requested=false 2024-11-16T09:58:11,594 DEBUG [RS_FLUSH_OPERATIONS-regionserver/037c58a19ddb:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegion(2603): Flush status journal for d9e28756f78b0555a6733ffbf6ab98bd: 2024-11-16T09:58:11,594 DEBUG [RS_FLUSH_OPERATIONS-regionserver/037c58a19ddb:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731751082172.d9e28756f78b0555a6733ffbf6ab98bd. 2024-11-16T09:58:11,594 DEBUG [RS_FLUSH_OPERATIONS-regionserver/037c58a19ddb:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=12 2024-11-16T09:58:11,595 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37877 {}] master.HMaster(4169): Remote procedure done, pid=12 2024-11-16T09:58:11,605 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=12, resume processing ppid=11 2024-11-16T09:58:11,605 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=12, ppid=11, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 809 msec 2024-11-16T09:58:11,610 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=11, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=11, table=TestAcidGuarantees in 821 msec 2024-11-16T09:58:11,617 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] regionserver.HRegion(8855): Flush requested on d9e28756f78b0555a6733ffbf6ab98bd 2024-11-16T09:58:11,617 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-16T09:58:11,617 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing d9e28756f78b0555a6733ffbf6ab98bd 3/3 column families, dataSize=80.51 KB heapSize=211.69 KB 2024-11-16T09:58:11,630 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/d9e28756f78b0555a6733ffbf6ab98bd/.tmp/A/4b3481cf62e34e3cbb47eb7e169c35c5 is 50, key is test_row_0/A:col10/1731751091615/Put/seqid=0 2024-11-16T09:58:11,650 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37591 is added to blk_1073741891_1067 (size=14741) 2024-11-16T09:58:11,651 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=271 (bloomFilter=true), to=hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/d9e28756f78b0555a6733ffbf6ab98bd/.tmp/A/4b3481cf62e34e3cbb47eb7e169c35c5 2024-11-16T09:58:11,657 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9e28756f78b0555a6733ffbf6ab98bd, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T09:58:11,657 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] ipc.CallRunner(138): callId: 76 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39696 deadline: 1731751151649, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9e28756f78b0555a6733ffbf6ab98bd, server=037c58a19ddb,37267,1731751077844 2024-11-16T09:58:11,658 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731751082172.d9e28756f78b0555a6733ffbf6ab98bd., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 , the old value is region=TestAcidGuarantees,,1731751082172.d9e28756f78b0555a6733ffbf6ab98bd., hostname=037c58a19ddb,37267,1731751077844, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9e28756f78b0555a6733ffbf6ab98bd, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:58:11,658 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9e28756f78b0555a6733ffbf6ab98bd, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T09:58:11,658 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731751082172.d9e28756f78b0555a6733ffbf6ab98bd., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9e28756f78b0555a6733ffbf6ab98bd, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:58:11,658 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731751082172.d9e28756f78b0555a6733ffbf6ab98bd., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 because the exception is null or not the one we care about 2024-11-16T09:58:11,658 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] ipc.CallRunner(138): callId: 74 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39638 deadline: 1731751151651, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9e28756f78b0555a6733ffbf6ab98bd, server=037c58a19ddb,37267,1731751077844 2024-11-16T09:58:11,659 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731751082172.d9e28756f78b0555a6733ffbf6ab98bd., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 , the old value is region=TestAcidGuarantees,,1731751082172.d9e28756f78b0555a6733ffbf6ab98bd., hostname=037c58a19ddb,37267,1731751077844, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9e28756f78b0555a6733ffbf6ab98bd, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:58:11,659 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731751082172.d9e28756f78b0555a6733ffbf6ab98bd., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9e28756f78b0555a6733ffbf6ab98bd, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:58:11,659 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731751082172.d9e28756f78b0555a6733ffbf6ab98bd., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 because the exception is null or not the one we care about 2024-11-16T09:58:11,660 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9e28756f78b0555a6733ffbf6ab98bd, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T09:58:11,661 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] ipc.CallRunner(138): callId: 82 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39654 deadline: 1731751151654, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9e28756f78b0555a6733ffbf6ab98bd, server=037c58a19ddb,37267,1731751077844 2024-11-16T09:58:11,661 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731751082172.d9e28756f78b0555a6733ffbf6ab98bd., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 , the old value is region=TestAcidGuarantees,,1731751082172.d9e28756f78b0555a6733ffbf6ab98bd., hostname=037c58a19ddb,37267,1731751077844, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9e28756f78b0555a6733ffbf6ab98bd, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:58:11,661 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731751082172.d9e28756f78b0555a6733ffbf6ab98bd., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9e28756f78b0555a6733ffbf6ab98bd, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:58:11,661 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731751082172.d9e28756f78b0555a6733ffbf6ab98bd., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 because the exception is null or not the one we care about 2024-11-16T09:58:11,663 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9e28756f78b0555a6733ffbf6ab98bd, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T09:58:11,663 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] ipc.CallRunner(138): callId: 74 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39630 deadline: 1731751151657, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9e28756f78b0555a6733ffbf6ab98bd, server=037c58a19ddb,37267,1731751077844 2024-11-16T09:58:11,664 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731751082172.d9e28756f78b0555a6733ffbf6ab98bd., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 , the old value is region=TestAcidGuarantees,,1731751082172.d9e28756f78b0555a6733ffbf6ab98bd., hostname=037c58a19ddb,37267,1731751077844, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9e28756f78b0555a6733ffbf6ab98bd, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:58:11,664 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9e28756f78b0555a6733ffbf6ab98bd, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T09:58:11,664 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731751082172.d9e28756f78b0555a6733ffbf6ab98bd., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9e28756f78b0555a6733ffbf6ab98bd, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:58:11,664 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731751082172.d9e28756f78b0555a6733ffbf6ab98bd., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 because the exception is null or not the one we care about 2024-11-16T09:58:11,664 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] ipc.CallRunner(138): callId: 81 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39620 deadline: 1731751151658, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9e28756f78b0555a6733ffbf6ab98bd, server=037c58a19ddb,37267,1731751077844 2024-11-16T09:58:11,664 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731751082172.d9e28756f78b0555a6733ffbf6ab98bd., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 , the old value is region=TestAcidGuarantees,,1731751082172.d9e28756f78b0555a6733ffbf6ab98bd., hostname=037c58a19ddb,37267,1731751077844, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9e28756f78b0555a6733ffbf6ab98bd, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:58:11,665 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731751082172.d9e28756f78b0555a6733ffbf6ab98bd., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9e28756f78b0555a6733ffbf6ab98bd, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:58:11,665 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731751082172.d9e28756f78b0555a6733ffbf6ab98bd., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 because the exception is null or not the one we care about 2024-11-16T09:58:11,670 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/d9e28756f78b0555a6733ffbf6ab98bd/.tmp/B/8a0b6de738244977a6688f5d91d6d3dc is 50, key is test_row_0/B:col10/1731751091615/Put/seqid=0 2024-11-16T09:58:11,688 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37591 is added to blk_1073741892_1068 (size=12301) 2024-11-16T09:58:11,691 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=271 (bloomFilter=true), to=hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/d9e28756f78b0555a6733ffbf6ab98bd/.tmp/B/8a0b6de738244977a6688f5d91d6d3dc 2024-11-16T09:58:11,704 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/d9e28756f78b0555a6733ffbf6ab98bd/.tmp/C/8b9ac2a67ae34acfab4cd9dcd7d9e194 is 50, key is test_row_0/C:col10/1731751091615/Put/seqid=0 2024-11-16T09:58:11,717 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37591 is added to blk_1073741893_1069 (size=12301) 2024-11-16T09:58:11,718 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=271 (bloomFilter=true), to=hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/d9e28756f78b0555a6733ffbf6ab98bd/.tmp/C/8b9ac2a67ae34acfab4cd9dcd7d9e194 2024-11-16T09:58:11,726 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/d9e28756f78b0555a6733ffbf6ab98bd/.tmp/A/4b3481cf62e34e3cbb47eb7e169c35c5 as hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/d9e28756f78b0555a6733ffbf6ab98bd/A/4b3481cf62e34e3cbb47eb7e169c35c5 2024-11-16T09:58:11,740 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/d9e28756f78b0555a6733ffbf6ab98bd/A/4b3481cf62e34e3cbb47eb7e169c35c5, entries=200, sequenceid=271, filesize=14.4 K 2024-11-16T09:58:11,741 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/d9e28756f78b0555a6733ffbf6ab98bd/.tmp/B/8a0b6de738244977a6688f5d91d6d3dc as hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/d9e28756f78b0555a6733ffbf6ab98bd/B/8a0b6de738244977a6688f5d91d6d3dc 2024-11-16T09:58:11,750 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/d9e28756f78b0555a6733ffbf6ab98bd/B/8a0b6de738244977a6688f5d91d6d3dc, entries=150, sequenceid=271, filesize=12.0 K 2024-11-16T09:58:11,751 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/d9e28756f78b0555a6733ffbf6ab98bd/.tmp/C/8b9ac2a67ae34acfab4cd9dcd7d9e194 as hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/d9e28756f78b0555a6733ffbf6ab98bd/C/8b9ac2a67ae34acfab4cd9dcd7d9e194 2024-11-16T09:58:11,756 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/d9e28756f78b0555a6733ffbf6ab98bd/C/8b9ac2a67ae34acfab4cd9dcd7d9e194, entries=150, sequenceid=271, filesize=12.0 K 2024-11-16T09:58:11,757 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~87.22 KB/89310, heapSize ~229.22 KB/234720, currentSize=120.76 KB/123660 for d9e28756f78b0555a6733ffbf6ab98bd in 140ms, sequenceid=271, compaction requested=true 2024-11-16T09:58:11,757 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for d9e28756f78b0555a6733ffbf6ab98bd: 2024-11-16T09:58:11,758 DEBUG [RS:0;037c58a19ddb:37267-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-16T09:58:11,758 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store d9e28756f78b0555a6733ffbf6ab98bd:A, priority=-2147483648, current under compaction store size is 2 2024-11-16T09:58:11,758 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-16T09:58:11,758 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store d9e28756f78b0555a6733ffbf6ab98bd:B, priority=-2147483648, current under compaction store size is 2 2024-11-16T09:58:11,758 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-16T09:58:11,758 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store d9e28756f78b0555a6733ffbf6ab98bd:C, priority=-2147483648, current under compaction store size is 3 2024-11-16T09:58:11,759 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=2), splitQueue=0 2024-11-16T09:58:11,759 DEBUG [RS:0;037c58a19ddb:37267-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 39589 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-16T09:58:11,759 DEBUG [RS:0;037c58a19ddb:37267-shortCompactions-0 {}] regionserver.HStore(1541): d9e28756f78b0555a6733ffbf6ab98bd/A is initiating minor compaction (all files) 2024-11-16T09:58:11,759 INFO [RS:0;037c58a19ddb:37267-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of d9e28756f78b0555a6733ffbf6ab98bd/A in TestAcidGuarantees,,1731751082172.d9e28756f78b0555a6733ffbf6ab98bd. 2024-11-16T09:58:11,759 INFO [RS:0;037c58a19ddb:37267-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/d9e28756f78b0555a6733ffbf6ab98bd/A/f18628f693944e64aadc099e58410220, hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/d9e28756f78b0555a6733ffbf6ab98bd/A/24edc08c1cf04c16bf00fef927979300, hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/d9e28756f78b0555a6733ffbf6ab98bd/A/4b3481cf62e34e3cbb47eb7e169c35c5] into tmpdir=hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/d9e28756f78b0555a6733ffbf6ab98bd/.tmp, totalSize=38.7 K 2024-11-16T09:58:11,760 DEBUG [RS:0;037c58a19ddb:37267-shortCompactions-0 {}] compactions.Compactor(225): Compacting f18628f693944e64aadc099e58410220, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=230, earliestPutTs=1731751090158 2024-11-16T09:58:11,761 DEBUG [RS:0;037c58a19ddb:37267-shortCompactions-0 {}] compactions.Compactor(225): Compacting 24edc08c1cf04c16bf00fef927979300, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=254, earliestPutTs=1731751090819 2024-11-16T09:58:11,761 DEBUG [RS:0;037c58a19ddb:37267-shortCompactions-0 {}] compactions.Compactor(225): Compacting 4b3481cf62e34e3cbb47eb7e169c35c5, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=271, earliestPutTs=1731751091501 2024-11-16T09:58:11,770 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-16T09:58:11,770 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing d9e28756f78b0555a6733ffbf6ab98bd 3/3 column families, dataSize=134.18 KB heapSize=352.31 KB 2024-11-16T09:58:11,770 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] regionserver.HRegion(8855): Flush requested on d9e28756f78b0555a6733ffbf6ab98bd 2024-11-16T09:58:11,775 INFO [RS:0;037c58a19ddb:37267-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): d9e28756f78b0555a6733ffbf6ab98bd#A#compaction#57 average throughput is 2.18 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-16T09:58:11,776 DEBUG [RS:0;037c58a19ddb:37267-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/d9e28756f78b0555a6733ffbf6ab98bd/.tmp/A/d737eb9df45049199ca0e28c20466aed is 50, key is test_row_0/A:col10/1731751091615/Put/seqid=0 2024-11-16T09:58:11,782 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/d9e28756f78b0555a6733ffbf6ab98bd/.tmp/A/3a30ee055c414e25a9cecba7e9183dbe is 50, key is test_row_0/A:col10/1731751091768/Put/seqid=0 2024-11-16T09:58:11,791 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37591 is added to blk_1073741894_1070 (size=12949) 2024-11-16T09:58:11,793 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37591 is added to blk_1073741895_1071 (size=12301) 2024-11-16T09:58:11,799 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9e28756f78b0555a6733ffbf6ab98bd, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T09:58:11,799 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] ipc.CallRunner(138): callId: 80 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39696 deadline: 1731751151791, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9e28756f78b0555a6733ffbf6ab98bd, server=037c58a19ddb,37267,1731751077844 2024-11-16T09:58:11,800 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731751082172.d9e28756f78b0555a6733ffbf6ab98bd., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 , the old value is region=TestAcidGuarantees,,1731751082172.d9e28756f78b0555a6733ffbf6ab98bd., hostname=037c58a19ddb,37267,1731751077844, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9e28756f78b0555a6733ffbf6ab98bd, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:58:11,800 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731751082172.d9e28756f78b0555a6733ffbf6ab98bd., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9e28756f78b0555a6733ffbf6ab98bd, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:58:11,800 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731751082172.d9e28756f78b0555a6733ffbf6ab98bd., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 because the exception is null or not the one we care about 2024-11-16T09:58:11,802 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9e28756f78b0555a6733ffbf6ab98bd, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T09:58:11,802 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] ipc.CallRunner(138): callId: 78 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39638 deadline: 1731751151797, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9e28756f78b0555a6733ffbf6ab98bd, server=037c58a19ddb,37267,1731751077844 2024-11-16T09:58:11,803 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9e28756f78b0555a6733ffbf6ab98bd, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T09:58:11,803 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] ipc.CallRunner(138): callId: 77 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39630 deadline: 1731751151797, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9e28756f78b0555a6733ffbf6ab98bd, server=037c58a19ddb,37267,1731751077844 2024-11-16T09:58:11,804 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9e28756f78b0555a6733ffbf6ab98bd, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T09:58:11,804 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] ipc.CallRunner(138): callId: 84 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39620 deadline: 1731751151801, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9e28756f78b0555a6733ffbf6ab98bd, server=037c58a19ddb,37267,1731751077844 2024-11-16T09:58:11,805 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731751082172.d9e28756f78b0555a6733ffbf6ab98bd., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 , the old value is region=TestAcidGuarantees,,1731751082172.d9e28756f78b0555a6733ffbf6ab98bd., hostname=037c58a19ddb,37267,1731751077844, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9e28756f78b0555a6733ffbf6ab98bd, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:58:11,805 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9e28756f78b0555a6733ffbf6ab98bd, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T09:58:11,805 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731751082172.d9e28756f78b0555a6733ffbf6ab98bd., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9e28756f78b0555a6733ffbf6ab98bd, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:58:11,805 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731751082172.d9e28756f78b0555a6733ffbf6ab98bd., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 because the exception is null or not the one we care about 2024-11-16T09:58:11,805 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] ipc.CallRunner(138): callId: 85 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39654 deadline: 1731751151802, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9e28756f78b0555a6733ffbf6ab98bd, server=037c58a19ddb,37267,1731751077844 2024-11-16T09:58:11,805 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731751082172.d9e28756f78b0555a6733ffbf6ab98bd., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 , the old value is region=TestAcidGuarantees,,1731751082172.d9e28756f78b0555a6733ffbf6ab98bd., hostname=037c58a19ddb,37267,1731751077844, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9e28756f78b0555a6733ffbf6ab98bd, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:58:11,805 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731751082172.d9e28756f78b0555a6733ffbf6ab98bd., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9e28756f78b0555a6733ffbf6ab98bd, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:58:11,805 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731751082172.d9e28756f78b0555a6733ffbf6ab98bd., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 because the exception is null or not the one we care about 2024-11-16T09:58:11,806 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731751082172.d9e28756f78b0555a6733ffbf6ab98bd., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 , the old value is region=TestAcidGuarantees,,1731751082172.d9e28756f78b0555a6733ffbf6ab98bd., hostname=037c58a19ddb,37267,1731751077844, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9e28756f78b0555a6733ffbf6ab98bd, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:58:11,806 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731751082172.d9e28756f78b0555a6733ffbf6ab98bd., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9e28756f78b0555a6733ffbf6ab98bd, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:58:11,806 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731751082172.d9e28756f78b0555a6733ffbf6ab98bd., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 because the exception is null or not the one we care about 2024-11-16T09:58:11,806 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731751082172.d9e28756f78b0555a6733ffbf6ab98bd., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 , the old value is region=TestAcidGuarantees,,1731751082172.d9e28756f78b0555a6733ffbf6ab98bd., hostname=037c58a19ddb,37267,1731751077844, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9e28756f78b0555a6733ffbf6ab98bd, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:58:11,806 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731751082172.d9e28756f78b0555a6733ffbf6ab98bd., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9e28756f78b0555a6733ffbf6ab98bd, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:58:11,807 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731751082172.d9e28756f78b0555a6733ffbf6ab98bd., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 because the exception is null or not the one we care about 2024-11-16T09:58:11,830 DEBUG [RS:0;037c58a19ddb:37267-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/d9e28756f78b0555a6733ffbf6ab98bd/.tmp/B/a93e2c2d05e94cf0b36912ca7912192a as hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/d9e28756f78b0555a6733ffbf6ab98bd/B/a93e2c2d05e94cf0b36912ca7912192a 2024-11-16T09:58:11,843 INFO [RS:0;037c58a19ddb:37267-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in d9e28756f78b0555a6733ffbf6ab98bd/B of d9e28756f78b0555a6733ffbf6ab98bd into a93e2c2d05e94cf0b36912ca7912192a(size=12.4 K), total size for store is 36.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-16T09:58:11,843 DEBUG [RS:0;037c58a19ddb:37267-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for d9e28756f78b0555a6733ffbf6ab98bd: 2024-11-16T09:58:11,843 INFO [RS:0;037c58a19ddb:37267-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731751082172.d9e28756f78b0555a6733ffbf6ab98bd., storeName=d9e28756f78b0555a6733ffbf6ab98bd/B, priority=13, startTime=1731751091345; duration=0sec 2024-11-16T09:58:11,843 DEBUG [RS:0;037c58a19ddb:37267-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=2), splitQueue=0 2024-11-16T09:58:11,843 DEBUG [RS:0;037c58a19ddb:37267-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: d9e28756f78b0555a6733ffbf6ab98bd:B 2024-11-16T09:58:11,843 DEBUG [RS:0;037c58a19ddb:37267-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: d9e28756f78b0555a6733ffbf6ab98bd:B 2024-11-16T09:58:11,843 DEBUG [RS:0;037c58a19ddb:37267-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-16T09:58:11,844 DEBUG [RS:0;037c58a19ddb:37267-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37149 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-16T09:58:11,845 DEBUG [RS:0;037c58a19ddb:37267-longCompactions-0 {}] regionserver.HStore(1541): d9e28756f78b0555a6733ffbf6ab98bd/C is initiating minor compaction (all files) 2024-11-16T09:58:11,845 INFO [RS:0;037c58a19ddb:37267-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of d9e28756f78b0555a6733ffbf6ab98bd/C in TestAcidGuarantees,,1731751082172.d9e28756f78b0555a6733ffbf6ab98bd. 2024-11-16T09:58:11,845 INFO [RS:0;037c58a19ddb:37267-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/d9e28756f78b0555a6733ffbf6ab98bd/C/79c45b5c5f474c3ea9417e42d3647139, hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/d9e28756f78b0555a6733ffbf6ab98bd/C/c8dd3aac45dc4fd3a068c59808acb2c5, hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/d9e28756f78b0555a6733ffbf6ab98bd/C/8b9ac2a67ae34acfab4cd9dcd7d9e194] into tmpdir=hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/d9e28756f78b0555a6733ffbf6ab98bd/.tmp, totalSize=36.3 K 2024-11-16T09:58:11,845 DEBUG [RS:0;037c58a19ddb:37267-longCompactions-0 {}] compactions.Compactor(225): Compacting 79c45b5c5f474c3ea9417e42d3647139, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=230, earliestPutTs=1731751090158 2024-11-16T09:58:11,846 DEBUG [RS:0;037c58a19ddb:37267-longCompactions-0 {}] compactions.Compactor(225): Compacting c8dd3aac45dc4fd3a068c59808acb2c5, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=254, earliestPutTs=1731751090819 2024-11-16T09:58:11,846 DEBUG [RS:0;037c58a19ddb:37267-longCompactions-0 {}] compactions.Compactor(225): Compacting 8b9ac2a67ae34acfab4cd9dcd7d9e194, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=271, earliestPutTs=1731751091501 2024-11-16T09:58:11,861 INFO [RS:0;037c58a19ddb:37267-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): d9e28756f78b0555a6733ffbf6ab98bd#C#compaction#59 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-16T09:58:11,861 DEBUG [RS:0;037c58a19ddb:37267-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/d9e28756f78b0555a6733ffbf6ab98bd/.tmp/C/36beb8f139a642b6b398f3dff65758fb is 50, key is test_row_0/C:col10/1731751091615/Put/seqid=0 2024-11-16T09:58:11,872 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37591 is added to blk_1073741896_1072 (size=12949) 2024-11-16T09:58:11,882 DEBUG [RS:0;037c58a19ddb:37267-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/d9e28756f78b0555a6733ffbf6ab98bd/.tmp/C/36beb8f139a642b6b398f3dff65758fb as hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/d9e28756f78b0555a6733ffbf6ab98bd/C/36beb8f139a642b6b398f3dff65758fb 2024-11-16T09:58:11,889 INFO [RS:0;037c58a19ddb:37267-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in d9e28756f78b0555a6733ffbf6ab98bd/C of d9e28756f78b0555a6733ffbf6ab98bd into 36beb8f139a642b6b398f3dff65758fb(size=12.6 K), total size for store is 12.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-16T09:58:11,889 DEBUG [RS:0;037c58a19ddb:37267-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for d9e28756f78b0555a6733ffbf6ab98bd: 2024-11-16T09:58:11,889 INFO [RS:0;037c58a19ddb:37267-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731751082172.d9e28756f78b0555a6733ffbf6ab98bd., storeName=d9e28756f78b0555a6733ffbf6ab98bd/C, priority=13, startTime=1731751091758; duration=0sec 2024-11-16T09:58:11,889 DEBUG [RS:0;037c58a19ddb:37267-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-16T09:58:11,889 DEBUG [RS:0;037c58a19ddb:37267-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: d9e28756f78b0555a6733ffbf6ab98bd:C 2024-11-16T09:58:11,890 DEBUG [RS:0;037c58a19ddb:37267-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-16T09:58:11,892 DEBUG [RS:0;037c58a19ddb:37267-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37149 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-16T09:58:11,892 DEBUG [RS:0;037c58a19ddb:37267-longCompactions-0 {}] regionserver.HStore(1541): d9e28756f78b0555a6733ffbf6ab98bd/B is initiating minor compaction (all files) 2024-11-16T09:58:11,892 INFO [RS:0;037c58a19ddb:37267-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of d9e28756f78b0555a6733ffbf6ab98bd/B in TestAcidGuarantees,,1731751082172.d9e28756f78b0555a6733ffbf6ab98bd. 2024-11-16T09:58:11,892 INFO [RS:0;037c58a19ddb:37267-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/d9e28756f78b0555a6733ffbf6ab98bd/B/a93e2c2d05e94cf0b36912ca7912192a, hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/d9e28756f78b0555a6733ffbf6ab98bd/B/836a5918e6164bf5a0d6607cf30b4271, hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/d9e28756f78b0555a6733ffbf6ab98bd/B/8a0b6de738244977a6688f5d91d6d3dc] into tmpdir=hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/d9e28756f78b0555a6733ffbf6ab98bd/.tmp, totalSize=36.3 K 2024-11-16T09:58:11,893 DEBUG [RS:0;037c58a19ddb:37267-longCompactions-0 {}] compactions.Compactor(225): Compacting a93e2c2d05e94cf0b36912ca7912192a, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=230, earliestPutTs=1731751090158 2024-11-16T09:58:11,893 DEBUG [RS:0;037c58a19ddb:37267-longCompactions-0 {}] compactions.Compactor(225): Compacting 836a5918e6164bf5a0d6607cf30b4271, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=254, earliestPutTs=1731751090819 2024-11-16T09:58:11,893 DEBUG [RS:0;037c58a19ddb:37267-longCompactions-0 {}] compactions.Compactor(225): Compacting 8a0b6de738244977a6688f5d91d6d3dc, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=271, earliestPutTs=1731751091501 2024-11-16T09:58:11,903 INFO [RS:0;037c58a19ddb:37267-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): d9e28756f78b0555a6733ffbf6ab98bd#B#compaction#60 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-16T09:58:11,904 DEBUG [RS:0;037c58a19ddb:37267-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/d9e28756f78b0555a6733ffbf6ab98bd/.tmp/B/a73b24e65a43405b9f959afc6671a804 is 50, key is test_row_0/B:col10/1731751091615/Put/seqid=0 2024-11-16T09:58:11,906 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9e28756f78b0555a6733ffbf6ab98bd, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T09:58:11,907 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] ipc.CallRunner(138): callId: 81 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39696 deadline: 1731751151903, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9e28756f78b0555a6733ffbf6ab98bd, server=037c58a19ddb,37267,1731751077844 2024-11-16T09:58:11,907 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731751082172.d9e28756f78b0555a6733ffbf6ab98bd., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 , the old value is region=TestAcidGuarantees,,1731751082172.d9e28756f78b0555a6733ffbf6ab98bd., hostname=037c58a19ddb,37267,1731751077844, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9e28756f78b0555a6733ffbf6ab98bd, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:58:11,907 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731751082172.d9e28756f78b0555a6733ffbf6ab98bd., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9e28756f78b0555a6733ffbf6ab98bd, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:58:11,907 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731751082172.d9e28756f78b0555a6733ffbf6ab98bd., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 because the exception is null or not the one we care about 2024-11-16T09:58:11,914 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37591 is added to blk_1073741897_1073 (size=12949) 2024-11-16T09:58:11,917 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9e28756f78b0555a6733ffbf6ab98bd, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T09:58:11,917 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] ipc.CallRunner(138): callId: 85 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39620 deadline: 1731751151914, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9e28756f78b0555a6733ffbf6ab98bd, server=037c58a19ddb,37267,1731751077844 2024-11-16T09:58:11,918 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731751082172.d9e28756f78b0555a6733ffbf6ab98bd., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 , the old value is region=TestAcidGuarantees,,1731751082172.d9e28756f78b0555a6733ffbf6ab98bd., hostname=037c58a19ddb,37267,1731751077844, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9e28756f78b0555a6733ffbf6ab98bd, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:58:11,918 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9e28756f78b0555a6733ffbf6ab98bd, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T09:58:11,918 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731751082172.d9e28756f78b0555a6733ffbf6ab98bd., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9e28756f78b0555a6733ffbf6ab98bd, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:58:11,918 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731751082172.d9e28756f78b0555a6733ffbf6ab98bd., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 because the exception is null or not the one we care about 2024-11-16T09:58:11,918 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] ipc.CallRunner(138): callId: 78 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39630 deadline: 1731751151914, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9e28756f78b0555a6733ffbf6ab98bd, server=037c58a19ddb,37267,1731751077844 2024-11-16T09:58:11,918 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9e28756f78b0555a6733ffbf6ab98bd, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T09:58:11,918 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731751082172.d9e28756f78b0555a6733ffbf6ab98bd., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 , the old value is region=TestAcidGuarantees,,1731751082172.d9e28756f78b0555a6733ffbf6ab98bd., hostname=037c58a19ddb,37267,1731751077844, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9e28756f78b0555a6733ffbf6ab98bd, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:58:11,919 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731751082172.d9e28756f78b0555a6733ffbf6ab98bd., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9e28756f78b0555a6733ffbf6ab98bd, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:58:11,919 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731751082172.d9e28756f78b0555a6733ffbf6ab98bd., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 because the exception is null or not the one we care about 2024-11-16T09:58:11,919 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] ipc.CallRunner(138): callId: 86 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39654 deadline: 1731751151914, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9e28756f78b0555a6733ffbf6ab98bd, server=037c58a19ddb,37267,1731751077844 2024-11-16T09:58:11,919 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9e28756f78b0555a6733ffbf6ab98bd, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T09:58:11,919 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] ipc.CallRunner(138): callId: 79 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39638 deadline: 1731751151914, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9e28756f78b0555a6733ffbf6ab98bd, server=037c58a19ddb,37267,1731751077844 2024-11-16T09:58:11,921 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731751082172.d9e28756f78b0555a6733ffbf6ab98bd., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 , the old value is region=TestAcidGuarantees,,1731751082172.d9e28756f78b0555a6733ffbf6ab98bd., hostname=037c58a19ddb,37267,1731751077844, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9e28756f78b0555a6733ffbf6ab98bd, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:58:11,922 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731751082172.d9e28756f78b0555a6733ffbf6ab98bd., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9e28756f78b0555a6733ffbf6ab98bd, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:58:11,922 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731751082172.d9e28756f78b0555a6733ffbf6ab98bd., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 because the exception is null or not the one we care about 2024-11-16T09:58:11,922 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731751082172.d9e28756f78b0555a6733ffbf6ab98bd., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 , the old value is region=TestAcidGuarantees,,1731751082172.d9e28756f78b0555a6733ffbf6ab98bd., hostname=037c58a19ddb,37267,1731751077844, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9e28756f78b0555a6733ffbf6ab98bd, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:58:11,922 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731751082172.d9e28756f78b0555a6733ffbf6ab98bd., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9e28756f78b0555a6733ffbf6ab98bd, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:58:11,922 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731751082172.d9e28756f78b0555a6733ffbf6ab98bd., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 because the exception is null or not the one we care about 2024-11-16T09:58:11,923 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37877 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=11 2024-11-16T09:58:11,924 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-16T09:58:11,924 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37877 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-16T09:58:11,925 DEBUG [RS:0;037c58a19ddb:37267-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/d9e28756f78b0555a6733ffbf6ab98bd/.tmp/B/a73b24e65a43405b9f959afc6671a804 as hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/d9e28756f78b0555a6733ffbf6ab98bd/B/a73b24e65a43405b9f959afc6671a804 2024-11-16T09:58:11,929 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37877 {}] procedure2.ProcedureExecutor(1139): Stored pid=13, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=13, table=TestAcidGuarantees 2024-11-16T09:58:11,930 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37877 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=13 2024-11-16T09:58:11,931 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=13, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=13, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-16T09:58:11,932 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=13, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=13, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-16T09:58:11,932 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=14, ppid=13, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-16T09:58:11,940 INFO [RS:0;037c58a19ddb:37267-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in d9e28756f78b0555a6733ffbf6ab98bd/B of d9e28756f78b0555a6733ffbf6ab98bd into a73b24e65a43405b9f959afc6671a804(size=12.6 K), total size for store is 12.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-16T09:58:11,941 DEBUG [RS:0;037c58a19ddb:37267-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for d9e28756f78b0555a6733ffbf6ab98bd: 2024-11-16T09:58:11,941 INFO [RS:0;037c58a19ddb:37267-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731751082172.d9e28756f78b0555a6733ffbf6ab98bd., storeName=d9e28756f78b0555a6733ffbf6ab98bd/B, priority=13, startTime=1731751091758; duration=0sec 2024-11-16T09:58:11,941 DEBUG [RS:0;037c58a19ddb:37267-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-16T09:58:11,941 DEBUG [RS:0;037c58a19ddb:37267-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: d9e28756f78b0555a6733ffbf6ab98bd:B 2024-11-16T09:58:12,032 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37877 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=13 2024-11-16T09:58:12,087 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=37267 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=14 2024-11-16T09:58:12,087 DEBUG [RS_FLUSH_OPERATIONS-regionserver/037c58a19ddb:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731751082172.d9e28756f78b0555a6733ffbf6ab98bd. 2024-11-16T09:58:12,088 DEBUG [RS_FLUSH_OPERATIONS-regionserver/037c58a19ddb:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731751082172.d9e28756f78b0555a6733ffbf6ab98bd. as already flushing 2024-11-16T09:58:12,088 DEBUG [RS_FLUSH_OPERATIONS-regionserver/037c58a19ddb:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731751082172.d9e28756f78b0555a6733ffbf6ab98bd. 2024-11-16T09:58:12,088 ERROR [RS_FLUSH_OPERATIONS-regionserver/037c58a19ddb:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] handler.RSProcedureHandler(58): pid=14 java.io.IOException: Unable to complete flush {ENCODED => d9e28756f78b0555a6733ffbf6ab98bd, NAME => 'TestAcidGuarantees,,1731751082172.d9e28756f78b0555a6733ffbf6ab98bd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T09:58:12,088 DEBUG [RS_FLUSH_OPERATIONS-regionserver/037c58a19ddb:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=14 java.io.IOException: Unable to complete flush {ENCODED => d9e28756f78b0555a6733ffbf6ab98bd, NAME => 'TestAcidGuarantees,,1731751082172.d9e28756f78b0555a6733ffbf6ab98bd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T09:58:12,089 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37877 {}] master.HMaster(4177): Remote procedure failed, pid=14 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d9e28756f78b0555a6733ffbf6ab98bd, NAME => 'TestAcidGuarantees,,1731751082172.d9e28756f78b0555a6733ffbf6ab98bd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d9e28756f78b0555a6733ffbf6ab98bd, NAME => 'TestAcidGuarantees,,1731751082172.d9e28756f78b0555a6733ffbf6ab98bd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T09:58:12,116 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9e28756f78b0555a6733ffbf6ab98bd, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T09:58:12,116 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] ipc.CallRunner(138): callId: 82 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39696 deadline: 1731751152113, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9e28756f78b0555a6733ffbf6ab98bd, server=037c58a19ddb,37267,1731751077844 2024-11-16T09:58:12,116 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731751082172.d9e28756f78b0555a6733ffbf6ab98bd., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 , the old value is region=TestAcidGuarantees,,1731751082172.d9e28756f78b0555a6733ffbf6ab98bd., hostname=037c58a19ddb,37267,1731751077844, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9e28756f78b0555a6733ffbf6ab98bd, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:58:12,117 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731751082172.d9e28756f78b0555a6733ffbf6ab98bd., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9e28756f78b0555a6733ffbf6ab98bd, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:58:12,117 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731751082172.d9e28756f78b0555a6733ffbf6ab98bd., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 because the exception is null or not the one we care about 2024-11-16T09:58:12,124 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9e28756f78b0555a6733ffbf6ab98bd, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T09:58:12,124 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] ipc.CallRunner(138): callId: 86 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39620 deadline: 1731751152124, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9e28756f78b0555a6733ffbf6ab98bd, server=037c58a19ddb,37267,1731751077844 2024-11-16T09:58:12,125 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731751082172.d9e28756f78b0555a6733ffbf6ab98bd., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 , the old value is region=TestAcidGuarantees,,1731751082172.d9e28756f78b0555a6733ffbf6ab98bd., hostname=037c58a19ddb,37267,1731751077844, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9e28756f78b0555a6733ffbf6ab98bd, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:58:12,125 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731751082172.d9e28756f78b0555a6733ffbf6ab98bd., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9e28756f78b0555a6733ffbf6ab98bd, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:58:12,125 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731751082172.d9e28756f78b0555a6733ffbf6ab98bd., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 because the exception is null or not the one we care about 2024-11-16T09:58:12,129 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9e28756f78b0555a6733ffbf6ab98bd, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T09:58:12,130 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] ipc.CallRunner(138): callId: 79 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39630 deadline: 1731751152129, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9e28756f78b0555a6733ffbf6ab98bd, server=037c58a19ddb,37267,1731751077844 2024-11-16T09:58:12,130 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731751082172.d9e28756f78b0555a6733ffbf6ab98bd., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 , the old value is region=TestAcidGuarantees,,1731751082172.d9e28756f78b0555a6733ffbf6ab98bd., hostname=037c58a19ddb,37267,1731751077844, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9e28756f78b0555a6733ffbf6ab98bd, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:58:12,130 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731751082172.d9e28756f78b0555a6733ffbf6ab98bd., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9e28756f78b0555a6733ffbf6ab98bd, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:58:12,131 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731751082172.d9e28756f78b0555a6733ffbf6ab98bd., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 because the exception is null or not the one we care about 2024-11-16T09:58:12,134 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9e28756f78b0555a6733ffbf6ab98bd, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T09:58:12,135 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] ipc.CallRunner(138): callId: 80 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39638 deadline: 1731751152133, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9e28756f78b0555a6733ffbf6ab98bd, server=037c58a19ddb,37267,1731751077844 2024-11-16T09:58:12,135 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9e28756f78b0555a6733ffbf6ab98bd, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T09:58:12,136 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] ipc.CallRunner(138): callId: 87 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39654 deadline: 1731751152133, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9e28756f78b0555a6733ffbf6ab98bd, server=037c58a19ddb,37267,1731751077844 2024-11-16T09:58:12,136 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731751082172.d9e28756f78b0555a6733ffbf6ab98bd., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 , the old value is region=TestAcidGuarantees,,1731751082172.d9e28756f78b0555a6733ffbf6ab98bd., hostname=037c58a19ddb,37267,1731751077844, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9e28756f78b0555a6733ffbf6ab98bd, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:58:12,136 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731751082172.d9e28756f78b0555a6733ffbf6ab98bd., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9e28756f78b0555a6733ffbf6ab98bd, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:58:12,136 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731751082172.d9e28756f78b0555a6733ffbf6ab98bd., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 because the exception is null or not the one we care about 2024-11-16T09:58:12,136 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731751082172.d9e28756f78b0555a6733ffbf6ab98bd., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 , the old value is region=TestAcidGuarantees,,1731751082172.d9e28756f78b0555a6733ffbf6ab98bd., hostname=037c58a19ddb,37267,1731751077844, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9e28756f78b0555a6733ffbf6ab98bd, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:58:12,136 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731751082172.d9e28756f78b0555a6733ffbf6ab98bd., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9e28756f78b0555a6733ffbf6ab98bd, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:58:12,136 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731751082172.d9e28756f78b0555a6733ffbf6ab98bd., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 because the exception is null or not the one we care about 2024-11-16T09:58:12,194 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=294 (bloomFilter=true), to=hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/d9e28756f78b0555a6733ffbf6ab98bd/.tmp/A/3a30ee055c414e25a9cecba7e9183dbe 2024-11-16T09:58:12,200 DEBUG [RS:0;037c58a19ddb:37267-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/d9e28756f78b0555a6733ffbf6ab98bd/.tmp/A/d737eb9df45049199ca0e28c20466aed as hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/d9e28756f78b0555a6733ffbf6ab98bd/A/d737eb9df45049199ca0e28c20466aed 2024-11-16T09:58:12,207 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/d9e28756f78b0555a6733ffbf6ab98bd/.tmp/B/0d5fa0a41a674039ac8d14c4f2a5924a is 50, key is test_row_0/B:col10/1731751091768/Put/seqid=0 2024-11-16T09:58:12,211 INFO [RS:0;037c58a19ddb:37267-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in d9e28756f78b0555a6733ffbf6ab98bd/A of d9e28756f78b0555a6733ffbf6ab98bd into d737eb9df45049199ca0e28c20466aed(size=12.6 K), total size for store is 12.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-16T09:58:12,211 DEBUG [RS:0;037c58a19ddb:37267-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for d9e28756f78b0555a6733ffbf6ab98bd: 2024-11-16T09:58:12,211 INFO [RS:0;037c58a19ddb:37267-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731751082172.d9e28756f78b0555a6733ffbf6ab98bd., storeName=d9e28756f78b0555a6733ffbf6ab98bd/A, priority=13, startTime=1731751091758; duration=0sec 2024-11-16T09:58:12,211 DEBUG [RS:0;037c58a19ddb:37267-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-16T09:58:12,211 DEBUG [RS:0;037c58a19ddb:37267-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: d9e28756f78b0555a6733ffbf6ab98bd:A 2024-11-16T09:58:12,215 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37591 is added to blk_1073741898_1074 (size=12301) 2024-11-16T09:58:12,242 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=37267 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=14 2024-11-16T09:58:12,242 DEBUG [RS_FLUSH_OPERATIONS-regionserver/037c58a19ddb:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731751082172.d9e28756f78b0555a6733ffbf6ab98bd. 2024-11-16T09:58:12,242 DEBUG [RS_FLUSH_OPERATIONS-regionserver/037c58a19ddb:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731751082172.d9e28756f78b0555a6733ffbf6ab98bd. as already flushing 2024-11-16T09:58:12,242 DEBUG [RS_FLUSH_OPERATIONS-regionserver/037c58a19ddb:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731751082172.d9e28756f78b0555a6733ffbf6ab98bd. 2024-11-16T09:58:12,242 ERROR [RS_FLUSH_OPERATIONS-regionserver/037c58a19ddb:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] handler.RSProcedureHandler(58): pid=14 java.io.IOException: Unable to complete flush {ENCODED => d9e28756f78b0555a6733ffbf6ab98bd, NAME => 'TestAcidGuarantees,,1731751082172.d9e28756f78b0555a6733ffbf6ab98bd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T09:58:12,243 DEBUG [RS_FLUSH_OPERATIONS-regionserver/037c58a19ddb:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=14 java.io.IOException: Unable to complete flush {ENCODED => d9e28756f78b0555a6733ffbf6ab98bd, NAME => 'TestAcidGuarantees,,1731751082172.d9e28756f78b0555a6733ffbf6ab98bd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T09:58:12,243 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37877 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=13 2024-11-16T09:58:12,243 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37877 {}] master.HMaster(4177): Remote procedure failed, pid=14 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d9e28756f78b0555a6733ffbf6ab98bd, NAME => 'TestAcidGuarantees,,1731751082172.d9e28756f78b0555a6733ffbf6ab98bd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d9e28756f78b0555a6733ffbf6ab98bd, NAME => 'TestAcidGuarantees,,1731751082172.d9e28756f78b0555a6733ffbf6ab98bd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T09:58:12,396 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=37267 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=14 2024-11-16T09:58:12,397 DEBUG [RS_FLUSH_OPERATIONS-regionserver/037c58a19ddb:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731751082172.d9e28756f78b0555a6733ffbf6ab98bd. 2024-11-16T09:58:12,397 DEBUG [RS_FLUSH_OPERATIONS-regionserver/037c58a19ddb:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731751082172.d9e28756f78b0555a6733ffbf6ab98bd. as already flushing 2024-11-16T09:58:12,397 DEBUG [RS_FLUSH_OPERATIONS-regionserver/037c58a19ddb:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731751082172.d9e28756f78b0555a6733ffbf6ab98bd. 2024-11-16T09:58:12,397 ERROR [RS_FLUSH_OPERATIONS-regionserver/037c58a19ddb:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] handler.RSProcedureHandler(58): pid=14 java.io.IOException: Unable to complete flush {ENCODED => d9e28756f78b0555a6733ffbf6ab98bd, NAME => 'TestAcidGuarantees,,1731751082172.d9e28756f78b0555a6733ffbf6ab98bd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T09:58:12,397 DEBUG [RS_FLUSH_OPERATIONS-regionserver/037c58a19ddb:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=14 java.io.IOException: Unable to complete flush {ENCODED => d9e28756f78b0555a6733ffbf6ab98bd, NAME => 'TestAcidGuarantees,,1731751082172.d9e28756f78b0555a6733ffbf6ab98bd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T09:58:12,398 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37877 {}] master.HMaster(4177): Remote procedure failed, pid=14 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d9e28756f78b0555a6733ffbf6ab98bd, NAME => 'TestAcidGuarantees,,1731751082172.d9e28756f78b0555a6733ffbf6ab98bd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d9e28756f78b0555a6733ffbf6ab98bd, NAME => 'TestAcidGuarantees,,1731751082172.d9e28756f78b0555a6733ffbf6ab98bd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T09:58:12,424 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9e28756f78b0555a6733ffbf6ab98bd, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T09:58:12,424 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] ipc.CallRunner(138): callId: 83 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39696 deadline: 1731751152423, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9e28756f78b0555a6733ffbf6ab98bd, server=037c58a19ddb,37267,1731751077844 2024-11-16T09:58:12,424 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731751082172.d9e28756f78b0555a6733ffbf6ab98bd., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 , the old value is region=TestAcidGuarantees,,1731751082172.d9e28756f78b0555a6733ffbf6ab98bd., hostname=037c58a19ddb,37267,1731751077844, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9e28756f78b0555a6733ffbf6ab98bd, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:58:12,425 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731751082172.d9e28756f78b0555a6733ffbf6ab98bd., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9e28756f78b0555a6733ffbf6ab98bd, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:58:12,425 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731751082172.d9e28756f78b0555a6733ffbf6ab98bd., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 because the exception is null or not the one we care about 2024-11-16T09:58:12,434 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9e28756f78b0555a6733ffbf6ab98bd, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T09:58:12,434 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] ipc.CallRunner(138): callId: 87 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39620 deadline: 1731751152433, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9e28756f78b0555a6733ffbf6ab98bd, server=037c58a19ddb,37267,1731751077844 2024-11-16T09:58:12,435 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731751082172.d9e28756f78b0555a6733ffbf6ab98bd., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 , the old value is region=TestAcidGuarantees,,1731751082172.d9e28756f78b0555a6733ffbf6ab98bd., hostname=037c58a19ddb,37267,1731751077844, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9e28756f78b0555a6733ffbf6ab98bd, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:58:12,435 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731751082172.d9e28756f78b0555a6733ffbf6ab98bd., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9e28756f78b0555a6733ffbf6ab98bd, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:58:12,435 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731751082172.d9e28756f78b0555a6733ffbf6ab98bd., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 because the exception is null or not the one we care about 2024-11-16T09:58:12,444 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9e28756f78b0555a6733ffbf6ab98bd, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T09:58:12,444 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] ipc.CallRunner(138): callId: 80 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39630 deadline: 1731751152443, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9e28756f78b0555a6733ffbf6ab98bd, server=037c58a19ddb,37267,1731751077844 2024-11-16T09:58:12,444 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731751082172.d9e28756f78b0555a6733ffbf6ab98bd., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 , the old value is region=TestAcidGuarantees,,1731751082172.d9e28756f78b0555a6733ffbf6ab98bd., hostname=037c58a19ddb,37267,1731751077844, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9e28756f78b0555a6733ffbf6ab98bd, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:58:12,445 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731751082172.d9e28756f78b0555a6733ffbf6ab98bd., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9e28756f78b0555a6733ffbf6ab98bd, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:58:12,444 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9e28756f78b0555a6733ffbf6ab98bd, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T09:58:12,445 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731751082172.d9e28756f78b0555a6733ffbf6ab98bd., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 because the exception is null or not the one we care about 2024-11-16T09:58:12,445 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] ipc.CallRunner(138): callId: 88 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39654 deadline: 1731751152444, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9e28756f78b0555a6733ffbf6ab98bd, server=037c58a19ddb,37267,1731751077844 2024-11-16T09:58:12,445 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731751082172.d9e28756f78b0555a6733ffbf6ab98bd., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 , the old value is region=TestAcidGuarantees,,1731751082172.d9e28756f78b0555a6733ffbf6ab98bd., hostname=037c58a19ddb,37267,1731751077844, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9e28756f78b0555a6733ffbf6ab98bd, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:58:12,445 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731751082172.d9e28756f78b0555a6733ffbf6ab98bd., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9e28756f78b0555a6733ffbf6ab98bd, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:58:12,445 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731751082172.d9e28756f78b0555a6733ffbf6ab98bd., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 because the exception is null or not the one we care about 2024-11-16T09:58:12,445 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9e28756f78b0555a6733ffbf6ab98bd, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T09:58:12,445 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] ipc.CallRunner(138): callId: 81 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39638 deadline: 1731751152444, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9e28756f78b0555a6733ffbf6ab98bd, server=037c58a19ddb,37267,1731751077844 2024-11-16T09:58:12,447 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731751082172.d9e28756f78b0555a6733ffbf6ab98bd., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 , the old value is region=TestAcidGuarantees,,1731751082172.d9e28756f78b0555a6733ffbf6ab98bd., hostname=037c58a19ddb,37267,1731751077844, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9e28756f78b0555a6733ffbf6ab98bd, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:58:12,447 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731751082172.d9e28756f78b0555a6733ffbf6ab98bd., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9e28756f78b0555a6733ffbf6ab98bd, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:58:12,447 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731751082172.d9e28756f78b0555a6733ffbf6ab98bd., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 because the exception is null or not the one we care about 2024-11-16T09:58:12,551 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=37267 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=14 2024-11-16T09:58:12,551 DEBUG [RS_FLUSH_OPERATIONS-regionserver/037c58a19ddb:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731751082172.d9e28756f78b0555a6733ffbf6ab98bd. 2024-11-16T09:58:12,551 DEBUG [RS_FLUSH_OPERATIONS-regionserver/037c58a19ddb:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731751082172.d9e28756f78b0555a6733ffbf6ab98bd. as already flushing 2024-11-16T09:58:12,551 DEBUG [RS_FLUSH_OPERATIONS-regionserver/037c58a19ddb:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731751082172.d9e28756f78b0555a6733ffbf6ab98bd. 2024-11-16T09:58:12,551 ERROR [RS_FLUSH_OPERATIONS-regionserver/037c58a19ddb:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] handler.RSProcedureHandler(58): pid=14 java.io.IOException: Unable to complete flush {ENCODED => d9e28756f78b0555a6733ffbf6ab98bd, NAME => 'TestAcidGuarantees,,1731751082172.d9e28756f78b0555a6733ffbf6ab98bd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T09:58:12,551 DEBUG [RS_FLUSH_OPERATIONS-regionserver/037c58a19ddb:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=14 java.io.IOException: Unable to complete flush {ENCODED => d9e28756f78b0555a6733ffbf6ab98bd, NAME => 'TestAcidGuarantees,,1731751082172.d9e28756f78b0555a6733ffbf6ab98bd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T09:58:12,552 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37877 {}] master.HMaster(4177): Remote procedure failed, pid=14 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d9e28756f78b0555a6733ffbf6ab98bd, NAME => 'TestAcidGuarantees,,1731751082172.d9e28756f78b0555a6733ffbf6ab98bd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d9e28756f78b0555a6733ffbf6ab98bd, NAME => 'TestAcidGuarantees,,1731751082172.d9e28756f78b0555a6733ffbf6ab98bd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T09:58:12,553 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37877 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=13 2024-11-16T09:58:12,616 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=294 (bloomFilter=true), to=hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/d9e28756f78b0555a6733ffbf6ab98bd/.tmp/B/0d5fa0a41a674039ac8d14c4f2a5924a 2024-11-16T09:58:12,627 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/d9e28756f78b0555a6733ffbf6ab98bd/.tmp/C/5058b1fc65bf44d6b262d1e0e572bc03 is 50, key is test_row_0/C:col10/1731751091768/Put/seqid=0 2024-11-16T09:58:12,632 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37591 is added to blk_1073741899_1075 (size=12301) 2024-11-16T09:58:12,705 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=37267 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=14 2024-11-16T09:58:12,705 DEBUG [RS_FLUSH_OPERATIONS-regionserver/037c58a19ddb:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731751082172.d9e28756f78b0555a6733ffbf6ab98bd. 2024-11-16T09:58:12,706 DEBUG [RS_FLUSH_OPERATIONS-regionserver/037c58a19ddb:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731751082172.d9e28756f78b0555a6733ffbf6ab98bd. as already flushing 2024-11-16T09:58:12,706 DEBUG [RS_FLUSH_OPERATIONS-regionserver/037c58a19ddb:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731751082172.d9e28756f78b0555a6733ffbf6ab98bd. 2024-11-16T09:58:12,706 ERROR [RS_FLUSH_OPERATIONS-regionserver/037c58a19ddb:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] handler.RSProcedureHandler(58): pid=14 java.io.IOException: Unable to complete flush {ENCODED => d9e28756f78b0555a6733ffbf6ab98bd, NAME => 'TestAcidGuarantees,,1731751082172.d9e28756f78b0555a6733ffbf6ab98bd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T09:58:12,706 DEBUG [RS_FLUSH_OPERATIONS-regionserver/037c58a19ddb:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=14 java.io.IOException: Unable to complete flush {ENCODED => d9e28756f78b0555a6733ffbf6ab98bd, NAME => 'TestAcidGuarantees,,1731751082172.d9e28756f78b0555a6733ffbf6ab98bd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T09:58:12,707 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37877 {}] master.HMaster(4177): Remote procedure failed, pid=14 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d9e28756f78b0555a6733ffbf6ab98bd, NAME => 'TestAcidGuarantees,,1731751082172.d9e28756f78b0555a6733ffbf6ab98bd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d9e28756f78b0555a6733ffbf6ab98bd, NAME => 'TestAcidGuarantees,,1731751082172.d9e28756f78b0555a6733ffbf6ab98bd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T09:58:12,860 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=37267 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=14 2024-11-16T09:58:12,860 DEBUG [RS_FLUSH_OPERATIONS-regionserver/037c58a19ddb:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731751082172.d9e28756f78b0555a6733ffbf6ab98bd. 2024-11-16T09:58:12,860 DEBUG [RS_FLUSH_OPERATIONS-regionserver/037c58a19ddb:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731751082172.d9e28756f78b0555a6733ffbf6ab98bd. as already flushing 2024-11-16T09:58:12,860 DEBUG [RS_FLUSH_OPERATIONS-regionserver/037c58a19ddb:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731751082172.d9e28756f78b0555a6733ffbf6ab98bd. 2024-11-16T09:58:12,860 ERROR [RS_FLUSH_OPERATIONS-regionserver/037c58a19ddb:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] handler.RSProcedureHandler(58): pid=14 java.io.IOException: Unable to complete flush {ENCODED => d9e28756f78b0555a6733ffbf6ab98bd, NAME => 'TestAcidGuarantees,,1731751082172.d9e28756f78b0555a6733ffbf6ab98bd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T09:58:12,860 DEBUG [RS_FLUSH_OPERATIONS-regionserver/037c58a19ddb:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=14 java.io.IOException: Unable to complete flush {ENCODED => d9e28756f78b0555a6733ffbf6ab98bd, NAME => 'TestAcidGuarantees,,1731751082172.d9e28756f78b0555a6733ffbf6ab98bd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T09:58:12,861 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37877 {}] master.HMaster(4177): Remote procedure failed, pid=14 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d9e28756f78b0555a6733ffbf6ab98bd, NAME => 'TestAcidGuarantees,,1731751082172.d9e28756f78b0555a6733ffbf6ab98bd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d9e28756f78b0555a6733ffbf6ab98bd, NAME => 'TestAcidGuarantees,,1731751082172.d9e28756f78b0555a6733ffbf6ab98bd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T09:58:12,935 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9e28756f78b0555a6733ffbf6ab98bd, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T09:58:12,935 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] ipc.CallRunner(138): callId: 84 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39696 deadline: 1731751152933, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9e28756f78b0555a6733ffbf6ab98bd, server=037c58a19ddb,37267,1731751077844 2024-11-16T09:58:12,936 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731751082172.d9e28756f78b0555a6733ffbf6ab98bd., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 , the old value is region=TestAcidGuarantees,,1731751082172.d9e28756f78b0555a6733ffbf6ab98bd., hostname=037c58a19ddb,37267,1731751077844, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9e28756f78b0555a6733ffbf6ab98bd, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:58:12,936 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731751082172.d9e28756f78b0555a6733ffbf6ab98bd., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9e28756f78b0555a6733ffbf6ab98bd, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:58:12,936 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731751082172.d9e28756f78b0555a6733ffbf6ab98bd., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 because the exception is null or not the one we care about 2024-11-16T09:58:12,945 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9e28756f78b0555a6733ffbf6ab98bd, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T09:58:12,945 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] ipc.CallRunner(138): callId: 88 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39620 deadline: 1731751152943, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9e28756f78b0555a6733ffbf6ab98bd, server=037c58a19ddb,37267,1731751077844 2024-11-16T09:58:12,947 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731751082172.d9e28756f78b0555a6733ffbf6ab98bd., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 , the old value is region=TestAcidGuarantees,,1731751082172.d9e28756f78b0555a6733ffbf6ab98bd., hostname=037c58a19ddb,37267,1731751077844, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9e28756f78b0555a6733ffbf6ab98bd, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:58:12,947 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731751082172.d9e28756f78b0555a6733ffbf6ab98bd., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9e28756f78b0555a6733ffbf6ab98bd, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:58:12,947 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731751082172.d9e28756f78b0555a6733ffbf6ab98bd., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 because the exception is null or not the one we care about 2024-11-16T09:58:12,953 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9e28756f78b0555a6733ffbf6ab98bd, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T09:58:12,954 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] ipc.CallRunner(138): callId: 89 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39654 deadline: 1731751152953, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9e28756f78b0555a6733ffbf6ab98bd, server=037c58a19ddb,37267,1731751077844 2024-11-16T09:58:12,954 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9e28756f78b0555a6733ffbf6ab98bd, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T09:58:12,954 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731751082172.d9e28756f78b0555a6733ffbf6ab98bd., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 , the old value is region=TestAcidGuarantees,,1731751082172.d9e28756f78b0555a6733ffbf6ab98bd., hostname=037c58a19ddb,37267,1731751077844, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9e28756f78b0555a6733ffbf6ab98bd, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:58:12,954 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731751082172.d9e28756f78b0555a6733ffbf6ab98bd., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9e28756f78b0555a6733ffbf6ab98bd, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:58:12,954 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] ipc.CallRunner(138): callId: 81 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39630 deadline: 1731751152953, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9e28756f78b0555a6733ffbf6ab98bd, server=037c58a19ddb,37267,1731751077844 2024-11-16T09:58:12,954 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731751082172.d9e28756f78b0555a6733ffbf6ab98bd., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 because the exception is null or not the one we care about 2024-11-16T09:58:12,955 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731751082172.d9e28756f78b0555a6733ffbf6ab98bd., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 , the old value is region=TestAcidGuarantees,,1731751082172.d9e28756f78b0555a6733ffbf6ab98bd., hostname=037c58a19ddb,37267,1731751077844, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9e28756f78b0555a6733ffbf6ab98bd, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:58:12,955 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731751082172.d9e28756f78b0555a6733ffbf6ab98bd., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9e28756f78b0555a6733ffbf6ab98bd, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:58:12,955 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731751082172.d9e28756f78b0555a6733ffbf6ab98bd., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 because the exception is null or not the one we care about 2024-11-16T09:58:12,960 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9e28756f78b0555a6733ffbf6ab98bd, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T09:58:12,961 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] ipc.CallRunner(138): callId: 82 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39638 deadline: 1731751152960, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9e28756f78b0555a6733ffbf6ab98bd, server=037c58a19ddb,37267,1731751077844 2024-11-16T09:58:12,962 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731751082172.d9e28756f78b0555a6733ffbf6ab98bd., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 , the old value is region=TestAcidGuarantees,,1731751082172.d9e28756f78b0555a6733ffbf6ab98bd., hostname=037c58a19ddb,37267,1731751077844, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9e28756f78b0555a6733ffbf6ab98bd, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:58:12,962 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731751082172.d9e28756f78b0555a6733ffbf6ab98bd., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9e28756f78b0555a6733ffbf6ab98bd, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:58:12,962 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731751082172.d9e28756f78b0555a6733ffbf6ab98bd., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 because the exception is null or not the one we care about 2024-11-16T09:58:13,014 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=37267 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=14 2024-11-16T09:58:13,014 DEBUG [RS_FLUSH_OPERATIONS-regionserver/037c58a19ddb:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731751082172.d9e28756f78b0555a6733ffbf6ab98bd. 2024-11-16T09:58:13,014 DEBUG [RS_FLUSH_OPERATIONS-regionserver/037c58a19ddb:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731751082172.d9e28756f78b0555a6733ffbf6ab98bd. as already flushing 2024-11-16T09:58:13,014 DEBUG [RS_FLUSH_OPERATIONS-regionserver/037c58a19ddb:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731751082172.d9e28756f78b0555a6733ffbf6ab98bd. 2024-11-16T09:58:13,014 ERROR [RS_FLUSH_OPERATIONS-regionserver/037c58a19ddb:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] handler.RSProcedureHandler(58): pid=14 java.io.IOException: Unable to complete flush {ENCODED => d9e28756f78b0555a6733ffbf6ab98bd, NAME => 'TestAcidGuarantees,,1731751082172.d9e28756f78b0555a6733ffbf6ab98bd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T09:58:13,014 DEBUG [RS_FLUSH_OPERATIONS-regionserver/037c58a19ddb:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=14 java.io.IOException: Unable to complete flush {ENCODED => d9e28756f78b0555a6733ffbf6ab98bd, NAME => 'TestAcidGuarantees,,1731751082172.d9e28756f78b0555a6733ffbf6ab98bd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T09:58:13,015 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37877 {}] master.HMaster(4177): Remote procedure failed, pid=14 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d9e28756f78b0555a6733ffbf6ab98bd, NAME => 'TestAcidGuarantees,,1731751082172.d9e28756f78b0555a6733ffbf6ab98bd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d9e28756f78b0555a6733ffbf6ab98bd, NAME => 'TestAcidGuarantees,,1731751082172.d9e28756f78b0555a6733ffbf6ab98bd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T09:58:13,033 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=294 (bloomFilter=true), to=hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/d9e28756f78b0555a6733ffbf6ab98bd/.tmp/C/5058b1fc65bf44d6b262d1e0e572bc03 2024-11-16T09:58:13,041 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/d9e28756f78b0555a6733ffbf6ab98bd/.tmp/A/3a30ee055c414e25a9cecba7e9183dbe as hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/d9e28756f78b0555a6733ffbf6ab98bd/A/3a30ee055c414e25a9cecba7e9183dbe 2024-11-16T09:58:13,050 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/d9e28756f78b0555a6733ffbf6ab98bd/A/3a30ee055c414e25a9cecba7e9183dbe, entries=150, sequenceid=294, filesize=12.0 K 2024-11-16T09:58:13,051 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/d9e28756f78b0555a6733ffbf6ab98bd/.tmp/B/0d5fa0a41a674039ac8d14c4f2a5924a as hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/d9e28756f78b0555a6733ffbf6ab98bd/B/0d5fa0a41a674039ac8d14c4f2a5924a 2024-11-16T09:58:13,062 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/d9e28756f78b0555a6733ffbf6ab98bd/B/0d5fa0a41a674039ac8d14c4f2a5924a, entries=150, sequenceid=294, filesize=12.0 K 2024-11-16T09:58:13,063 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37877 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=13 2024-11-16T09:58:13,063 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/d9e28756f78b0555a6733ffbf6ab98bd/.tmp/C/5058b1fc65bf44d6b262d1e0e572bc03 as hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/d9e28756f78b0555a6733ffbf6ab98bd/C/5058b1fc65bf44d6b262d1e0e572bc03 2024-11-16T09:58:13,071 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/d9e28756f78b0555a6733ffbf6ab98bd/C/5058b1fc65bf44d6b262d1e0e572bc03, entries=150, sequenceid=294, filesize=12.0 K 2024-11-16T09:58:13,072 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~134.18 KB/137400, heapSize ~352.27 KB/360720, currentSize=67.09 KB/68700 for d9e28756f78b0555a6733ffbf6ab98bd in 1302ms, sequenceid=294, compaction requested=false 2024-11-16T09:58:13,072 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for d9e28756f78b0555a6733ffbf6ab98bd: 2024-11-16T09:58:13,168 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=37267 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=14 2024-11-16T09:58:13,168 DEBUG [RS_FLUSH_OPERATIONS-regionserver/037c58a19ddb:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731751082172.d9e28756f78b0555a6733ffbf6ab98bd. 2024-11-16T09:58:13,169 INFO [RS_FLUSH_OPERATIONS-regionserver/037c58a19ddb:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegion(2902): Flushing d9e28756f78b0555a6733ffbf6ab98bd 3/3 column families, dataSize=67.09 KB heapSize=176.53 KB 2024-11-16T09:58:13,187 DEBUG [RS_FLUSH_OPERATIONS-regionserver/037c58a19ddb:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/d9e28756f78b0555a6733ffbf6ab98bd/.tmp/A/f2ad538a347b4ca9bf7afd4e658e5bac is 50, key is test_row_0/A:col10/1731751091797/Put/seqid=0 2024-11-16T09:58:13,202 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37591 is added to blk_1073741900_1076 (size=12301) 2024-11-16T09:58:13,603 INFO [RS_FLUSH_OPERATIONS-regionserver/037c58a19ddb:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=311 (bloomFilter=true), to=hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/d9e28756f78b0555a6733ffbf6ab98bd/.tmp/A/f2ad538a347b4ca9bf7afd4e658e5bac 2024-11-16T09:58:13,614 DEBUG [RS_FLUSH_OPERATIONS-regionserver/037c58a19ddb:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/d9e28756f78b0555a6733ffbf6ab98bd/.tmp/B/5981e9c45ed3447bacfe73467c7101e9 is 50, key is test_row_0/B:col10/1731751091797/Put/seqid=0 2024-11-16T09:58:13,627 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37591 is added to blk_1073741901_1077 (size=12301) 2024-11-16T09:58:13,946 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] regionserver.HRegion(8855): Flush requested on d9e28756f78b0555a6733ffbf6ab98bd 2024-11-16T09:58:13,946 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731751082172.d9e28756f78b0555a6733ffbf6ab98bd. as already flushing 2024-11-16T09:58:13,986 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9e28756f78b0555a6733ffbf6ab98bd, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T09:58:13,986 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] ipc.CallRunner(138): callId: 83 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39630 deadline: 1731751153981, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9e28756f78b0555a6733ffbf6ab98bd, server=037c58a19ddb,37267,1731751077844 2024-11-16T09:58:13,986 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9e28756f78b0555a6733ffbf6ab98bd, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T09:58:13,986 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731751082172.d9e28756f78b0555a6733ffbf6ab98bd., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 , the old value is region=TestAcidGuarantees,,1731751082172.d9e28756f78b0555a6733ffbf6ab98bd., hostname=037c58a19ddb,37267,1731751077844, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9e28756f78b0555a6733ffbf6ab98bd, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:58:13,987 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731751082172.d9e28756f78b0555a6733ffbf6ab98bd., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9e28756f78b0555a6733ffbf6ab98bd, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:58:13,987 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] ipc.CallRunner(138): callId: 84 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39638 deadline: 1731751153982, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9e28756f78b0555a6733ffbf6ab98bd, server=037c58a19ddb,37267,1731751077844 2024-11-16T09:58:13,987 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731751082172.d9e28756f78b0555a6733ffbf6ab98bd., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 because the exception is null or not the one we care about 2024-11-16T09:58:13,987 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9e28756f78b0555a6733ffbf6ab98bd, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T09:58:13,987 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731751082172.d9e28756f78b0555a6733ffbf6ab98bd., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 , the old value is region=TestAcidGuarantees,,1731751082172.d9e28756f78b0555a6733ffbf6ab98bd., hostname=037c58a19ddb,37267,1731751077844, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9e28756f78b0555a6733ffbf6ab98bd, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:58:13,987 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731751082172.d9e28756f78b0555a6733ffbf6ab98bd., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9e28756f78b0555a6733ffbf6ab98bd, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:58:13,987 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] ipc.CallRunner(138): callId: 93 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39696 deadline: 1731751153982, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9e28756f78b0555a6733ffbf6ab98bd, server=037c58a19ddb,37267,1731751077844 2024-11-16T09:58:13,987 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731751082172.d9e28756f78b0555a6733ffbf6ab98bd., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 because the exception is null or not the one we care about 2024-11-16T09:58:13,987 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9e28756f78b0555a6733ffbf6ab98bd, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T09:58:13,987 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] ipc.CallRunner(138): callId: 95 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39620 deadline: 1731751153983, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9e28756f78b0555a6733ffbf6ab98bd, server=037c58a19ddb,37267,1731751077844 2024-11-16T09:58:13,987 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731751082172.d9e28756f78b0555a6733ffbf6ab98bd., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 , the old value is region=TestAcidGuarantees,,1731751082172.d9e28756f78b0555a6733ffbf6ab98bd., hostname=037c58a19ddb,37267,1731751077844, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9e28756f78b0555a6733ffbf6ab98bd, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:58:13,988 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731751082172.d9e28756f78b0555a6733ffbf6ab98bd., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9e28756f78b0555a6733ffbf6ab98bd, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:58:13,988 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731751082172.d9e28756f78b0555a6733ffbf6ab98bd., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 because the exception is null or not the one we care about 2024-11-16T09:58:13,988 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9e28756f78b0555a6733ffbf6ab98bd, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T09:58:13,988 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] ipc.CallRunner(138): callId: 94 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39654 deadline: 1731751153986, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9e28756f78b0555a6733ffbf6ab98bd, server=037c58a19ddb,37267,1731751077844 2024-11-16T09:58:13,988 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731751082172.d9e28756f78b0555a6733ffbf6ab98bd., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 , the old value is region=TestAcidGuarantees,,1731751082172.d9e28756f78b0555a6733ffbf6ab98bd., hostname=037c58a19ddb,37267,1731751077844, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9e28756f78b0555a6733ffbf6ab98bd, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:58:13,988 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731751082172.d9e28756f78b0555a6733ffbf6ab98bd., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9e28756f78b0555a6733ffbf6ab98bd, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:58:13,988 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731751082172.d9e28756f78b0555a6733ffbf6ab98bd., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 because the exception is null or not the one we care about 2024-11-16T09:58:13,988 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731751082172.d9e28756f78b0555a6733ffbf6ab98bd., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 , the old value is region=TestAcidGuarantees,,1731751082172.d9e28756f78b0555a6733ffbf6ab98bd., hostname=037c58a19ddb,37267,1731751077844, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9e28756f78b0555a6733ffbf6ab98bd, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:58:13,988 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731751082172.d9e28756f78b0555a6733ffbf6ab98bd., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9e28756f78b0555a6733ffbf6ab98bd, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:58:13,988 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731751082172.d9e28756f78b0555a6733ffbf6ab98bd., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 because the exception is null or not the one we care about 2024-11-16T09:58:14,028 INFO [RS_FLUSH_OPERATIONS-regionserver/037c58a19ddb:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=311 (bloomFilter=true), to=hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/d9e28756f78b0555a6733ffbf6ab98bd/.tmp/B/5981e9c45ed3447bacfe73467c7101e9 2024-11-16T09:58:14,039 DEBUG [RS_FLUSH_OPERATIONS-regionserver/037c58a19ddb:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/d9e28756f78b0555a6733ffbf6ab98bd/.tmp/C/d869628b6caf43adb4766aab62057038 is 50, key is test_row_0/C:col10/1731751091797/Put/seqid=0 2024-11-16T09:58:14,044 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37591 is added to blk_1073741902_1078 (size=12301) 2024-11-16T09:58:14,073 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37877 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=13 2024-11-16T09:58:14,095 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9e28756f78b0555a6733ffbf6ab98bd, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T09:58:14,096 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] ipc.CallRunner(138): callId: 84 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39630 deadline: 1731751154093, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9e28756f78b0555a6733ffbf6ab98bd, server=037c58a19ddb,37267,1731751077844 2024-11-16T09:58:14,096 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731751082172.d9e28756f78b0555a6733ffbf6ab98bd., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 , the old value is region=TestAcidGuarantees,,1731751082172.d9e28756f78b0555a6733ffbf6ab98bd., hostname=037c58a19ddb,37267,1731751077844, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9e28756f78b0555a6733ffbf6ab98bd, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:58:14,096 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731751082172.d9e28756f78b0555a6733ffbf6ab98bd., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9e28756f78b0555a6733ffbf6ab98bd, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:58:14,096 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9e28756f78b0555a6733ffbf6ab98bd, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T09:58:14,096 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731751082172.d9e28756f78b0555a6733ffbf6ab98bd., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 because the exception is null or not the one we care about 2024-11-16T09:58:14,096 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] ipc.CallRunner(138): callId: 85 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39638 deadline: 1731751154093, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9e28756f78b0555a6733ffbf6ab98bd, server=037c58a19ddb,37267,1731751077844 2024-11-16T09:58:14,097 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9e28756f78b0555a6733ffbf6ab98bd, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T09:58:14,097 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] ipc.CallRunner(138): callId: 94 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39696 deadline: 1731751154094, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9e28756f78b0555a6733ffbf6ab98bd, server=037c58a19ddb,37267,1731751077844 2024-11-16T09:58:14,097 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731751082172.d9e28756f78b0555a6733ffbf6ab98bd., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 , the old value is region=TestAcidGuarantees,,1731751082172.d9e28756f78b0555a6733ffbf6ab98bd., hostname=037c58a19ddb,37267,1731751077844, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9e28756f78b0555a6733ffbf6ab98bd, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:58:14,097 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731751082172.d9e28756f78b0555a6733ffbf6ab98bd., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9e28756f78b0555a6733ffbf6ab98bd, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:58:14,097 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9e28756f78b0555a6733ffbf6ab98bd, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T09:58:14,097 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731751082172.d9e28756f78b0555a6733ffbf6ab98bd., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 because the exception is null or not the one we care about 2024-11-16T09:58:14,097 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] ipc.CallRunner(138): callId: 96 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39620 deadline: 1731751154094, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9e28756f78b0555a6733ffbf6ab98bd, server=037c58a19ddb,37267,1731751077844 2024-11-16T09:58:14,097 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9e28756f78b0555a6733ffbf6ab98bd, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T09:58:14,097 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] ipc.CallRunner(138): callId: 95 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39654 deadline: 1731751154094, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9e28756f78b0555a6733ffbf6ab98bd, server=037c58a19ddb,37267,1731751077844 2024-11-16T09:58:14,097 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731751082172.d9e28756f78b0555a6733ffbf6ab98bd., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 , the old value is region=TestAcidGuarantees,,1731751082172.d9e28756f78b0555a6733ffbf6ab98bd., hostname=037c58a19ddb,37267,1731751077844, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9e28756f78b0555a6733ffbf6ab98bd, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:58:14,097 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731751082172.d9e28756f78b0555a6733ffbf6ab98bd., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9e28756f78b0555a6733ffbf6ab98bd, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:58:14,097 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731751082172.d9e28756f78b0555a6733ffbf6ab98bd., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 , the old value is region=TestAcidGuarantees,,1731751082172.d9e28756f78b0555a6733ffbf6ab98bd., hostname=037c58a19ddb,37267,1731751077844, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9e28756f78b0555a6733ffbf6ab98bd, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:58:14,097 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731751082172.d9e28756f78b0555a6733ffbf6ab98bd., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 because the exception is null or not the one we care about 2024-11-16T09:58:14,098 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731751082172.d9e28756f78b0555a6733ffbf6ab98bd., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9e28756f78b0555a6733ffbf6ab98bd, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:58:14,098 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731751082172.d9e28756f78b0555a6733ffbf6ab98bd., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 because the exception is null or not the one we care about 2024-11-16T09:58:14,098 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731751082172.d9e28756f78b0555a6733ffbf6ab98bd., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 , the old value is region=TestAcidGuarantees,,1731751082172.d9e28756f78b0555a6733ffbf6ab98bd., hostname=037c58a19ddb,37267,1731751077844, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9e28756f78b0555a6733ffbf6ab98bd, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:58:14,098 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731751082172.d9e28756f78b0555a6733ffbf6ab98bd., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9e28756f78b0555a6733ffbf6ab98bd, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:58:14,098 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731751082172.d9e28756f78b0555a6733ffbf6ab98bd., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 because the exception is null or not the one we care about 2024-11-16T09:58:14,305 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9e28756f78b0555a6733ffbf6ab98bd, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T09:58:14,306 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] ipc.CallRunner(138): callId: 85 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39630 deadline: 1731751154303, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9e28756f78b0555a6733ffbf6ab98bd, server=037c58a19ddb,37267,1731751077844 2024-11-16T09:58:14,306 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731751082172.d9e28756f78b0555a6733ffbf6ab98bd., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 , the old value is region=TestAcidGuarantees,,1731751082172.d9e28756f78b0555a6733ffbf6ab98bd., hostname=037c58a19ddb,37267,1731751077844, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9e28756f78b0555a6733ffbf6ab98bd, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:58:14,307 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731751082172.d9e28756f78b0555a6733ffbf6ab98bd., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9e28756f78b0555a6733ffbf6ab98bd, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:58:14,307 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731751082172.d9e28756f78b0555a6733ffbf6ab98bd., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 because the exception is null or not the one we care about 2024-11-16T09:58:14,307 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9e28756f78b0555a6733ffbf6ab98bd, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T09:58:14,307 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] ipc.CallRunner(138): callId: 86 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39638 deadline: 1731751154303, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9e28756f78b0555a6733ffbf6ab98bd, server=037c58a19ddb,37267,1731751077844 2024-11-16T09:58:14,307 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731751082172.d9e28756f78b0555a6733ffbf6ab98bd., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 , the old value is region=TestAcidGuarantees,,1731751082172.d9e28756f78b0555a6733ffbf6ab98bd., hostname=037c58a19ddb,37267,1731751077844, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9e28756f78b0555a6733ffbf6ab98bd, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:58:14,307 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731751082172.d9e28756f78b0555a6733ffbf6ab98bd., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9e28756f78b0555a6733ffbf6ab98bd, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:58:14,307 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731751082172.d9e28756f78b0555a6733ffbf6ab98bd., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 because the exception is null or not the one we care about 2024-11-16T09:58:14,311 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9e28756f78b0555a6733ffbf6ab98bd, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T09:58:14,311 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] ipc.CallRunner(138): callId: 95 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39696 deadline: 1731751154303, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9e28756f78b0555a6733ffbf6ab98bd, server=037c58a19ddb,37267,1731751077844 2024-11-16T09:58:14,311 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9e28756f78b0555a6733ffbf6ab98bd, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T09:58:14,311 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] ipc.CallRunner(138): callId: 97 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39620 deadline: 1731751154303, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9e28756f78b0555a6733ffbf6ab98bd, server=037c58a19ddb,37267,1731751077844 2024-11-16T09:58:14,311 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9e28756f78b0555a6733ffbf6ab98bd, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T09:58:14,312 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] ipc.CallRunner(138): callId: 96 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39654 deadline: 1731751154303, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9e28756f78b0555a6733ffbf6ab98bd, server=037c58a19ddb,37267,1731751077844 2024-11-16T09:58:14,312 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731751082172.d9e28756f78b0555a6733ffbf6ab98bd., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 , the old value is region=TestAcidGuarantees,,1731751082172.d9e28756f78b0555a6733ffbf6ab98bd., hostname=037c58a19ddb,37267,1731751077844, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9e28756f78b0555a6733ffbf6ab98bd, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:58:14,312 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731751082172.d9e28756f78b0555a6733ffbf6ab98bd., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9e28756f78b0555a6733ffbf6ab98bd, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:58:14,312 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731751082172.d9e28756f78b0555a6733ffbf6ab98bd., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 because the exception is null or not the one we care about 2024-11-16T09:58:14,312 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731751082172.d9e28756f78b0555a6733ffbf6ab98bd., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 , the old value is region=TestAcidGuarantees,,1731751082172.d9e28756f78b0555a6733ffbf6ab98bd., hostname=037c58a19ddb,37267,1731751077844, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9e28756f78b0555a6733ffbf6ab98bd, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:58:14,312 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731751082172.d9e28756f78b0555a6733ffbf6ab98bd., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 , the old value is region=TestAcidGuarantees,,1731751082172.d9e28756f78b0555a6733ffbf6ab98bd., hostname=037c58a19ddb,37267,1731751077844, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9e28756f78b0555a6733ffbf6ab98bd, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:58:14,312 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731751082172.d9e28756f78b0555a6733ffbf6ab98bd., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9e28756f78b0555a6733ffbf6ab98bd, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:58:14,312 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731751082172.d9e28756f78b0555a6733ffbf6ab98bd., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9e28756f78b0555a6733ffbf6ab98bd, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:58:14,312 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731751082172.d9e28756f78b0555a6733ffbf6ab98bd., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 because the exception is null or not the one we care about 2024-11-16T09:58:14,312 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731751082172.d9e28756f78b0555a6733ffbf6ab98bd., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 because the exception is null or not the one we care about 2024-11-16T09:58:14,445 INFO [RS_FLUSH_OPERATIONS-regionserver/037c58a19ddb:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=311 (bloomFilter=true), to=hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/d9e28756f78b0555a6733ffbf6ab98bd/.tmp/C/d869628b6caf43adb4766aab62057038 2024-11-16T09:58:14,451 DEBUG [RS_FLUSH_OPERATIONS-regionserver/037c58a19ddb:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/d9e28756f78b0555a6733ffbf6ab98bd/.tmp/A/f2ad538a347b4ca9bf7afd4e658e5bac as hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/d9e28756f78b0555a6733ffbf6ab98bd/A/f2ad538a347b4ca9bf7afd4e658e5bac 2024-11-16T09:58:14,457 INFO [RS_FLUSH_OPERATIONS-regionserver/037c58a19ddb:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/d9e28756f78b0555a6733ffbf6ab98bd/A/f2ad538a347b4ca9bf7afd4e658e5bac, entries=150, sequenceid=311, filesize=12.0 K 2024-11-16T09:58:14,458 DEBUG [RS_FLUSH_OPERATIONS-regionserver/037c58a19ddb:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/d9e28756f78b0555a6733ffbf6ab98bd/.tmp/B/5981e9c45ed3447bacfe73467c7101e9 as hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/d9e28756f78b0555a6733ffbf6ab98bd/B/5981e9c45ed3447bacfe73467c7101e9 2024-11-16T09:58:14,464 INFO [RS_FLUSH_OPERATIONS-regionserver/037c58a19ddb:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/d9e28756f78b0555a6733ffbf6ab98bd/B/5981e9c45ed3447bacfe73467c7101e9, entries=150, sequenceid=311, filesize=12.0 K 2024-11-16T09:58:14,465 DEBUG [RS_FLUSH_OPERATIONS-regionserver/037c58a19ddb:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/d9e28756f78b0555a6733ffbf6ab98bd/.tmp/C/d869628b6caf43adb4766aab62057038 as hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/d9e28756f78b0555a6733ffbf6ab98bd/C/d869628b6caf43adb4766aab62057038 2024-11-16T09:58:14,476 INFO [RS_FLUSH_OPERATIONS-regionserver/037c58a19ddb:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/d9e28756f78b0555a6733ffbf6ab98bd/C/d869628b6caf43adb4766aab62057038, entries=150, sequenceid=311, filesize=12.0 K 2024-11-16T09:58:14,478 INFO [RS_FLUSH_OPERATIONS-regionserver/037c58a19ddb:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegion(3140): Finished flush of dataSize ~67.09 KB/68700, heapSize ~176.48 KB/180720, currentSize=134.18 KB/137400 for d9e28756f78b0555a6733ffbf6ab98bd in 1309ms, sequenceid=311, compaction requested=true 2024-11-16T09:58:14,478 DEBUG [RS_FLUSH_OPERATIONS-regionserver/037c58a19ddb:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegion(2603): Flush status journal for d9e28756f78b0555a6733ffbf6ab98bd: 2024-11-16T09:58:14,478 DEBUG [RS_FLUSH_OPERATIONS-regionserver/037c58a19ddb:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731751082172.d9e28756f78b0555a6733ffbf6ab98bd. 2024-11-16T09:58:14,478 DEBUG [RS_FLUSH_OPERATIONS-regionserver/037c58a19ddb:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=14 2024-11-16T09:58:14,479 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37877 {}] master.HMaster(4169): Remote procedure done, pid=14 2024-11-16T09:58:14,483 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=14, resume processing ppid=13 2024-11-16T09:58:14,483 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=14, ppid=13, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.5480 sec 2024-11-16T09:58:14,488 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=13, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=13, table=TestAcidGuarantees in 2.5630 sec 2024-11-16T09:58:14,617 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] regionserver.HRegion(8855): Flush requested on d9e28756f78b0555a6733ffbf6ab98bd 2024-11-16T09:58:14,617 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-16T09:58:14,617 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing d9e28756f78b0555a6733ffbf6ab98bd 3/3 column families, dataSize=147.60 KB heapSize=387.47 KB 2024-11-16T09:58:14,623 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/d9e28756f78b0555a6733ffbf6ab98bd/.tmp/A/18344b8f21c345efa4b177f11720008e is 50, key is test_row_0/A:col10/1731751094615/Put/seqid=0 2024-11-16T09:58:14,628 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37591 is added to blk_1073741903_1079 (size=17181) 2024-11-16T09:58:14,639 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9e28756f78b0555a6733ffbf6ab98bd, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T09:58:14,639 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9e28756f78b0555a6733ffbf6ab98bd, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T09:58:14,640 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] ipc.CallRunner(138): callId: 90 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39638 deadline: 1731751154634, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9e28756f78b0555a6733ffbf6ab98bd, server=037c58a19ddb,37267,1731751077844 2024-11-16T09:58:14,640 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] ipc.CallRunner(138): callId: 99 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39620 deadline: 1731751154634, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9e28756f78b0555a6733ffbf6ab98bd, server=037c58a19ddb,37267,1731751077844 2024-11-16T09:58:14,640 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731751082172.d9e28756f78b0555a6733ffbf6ab98bd., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 , the old value is region=TestAcidGuarantees,,1731751082172.d9e28756f78b0555a6733ffbf6ab98bd., hostname=037c58a19ddb,37267,1731751077844, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9e28756f78b0555a6733ffbf6ab98bd, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:58:14,640 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9e28756f78b0555a6733ffbf6ab98bd, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T09:58:14,640 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731751082172.d9e28756f78b0555a6733ffbf6ab98bd., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 , the old value is region=TestAcidGuarantees,,1731751082172.d9e28756f78b0555a6733ffbf6ab98bd., hostname=037c58a19ddb,37267,1731751077844, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9e28756f78b0555a6733ffbf6ab98bd, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:58:14,640 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731751082172.d9e28756f78b0555a6733ffbf6ab98bd., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9e28756f78b0555a6733ffbf6ab98bd, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:58:14,640 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731751082172.d9e28756f78b0555a6733ffbf6ab98bd., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 because the exception is null or not the one we care about 2024-11-16T09:58:14,640 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731751082172.d9e28756f78b0555a6733ffbf6ab98bd., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9e28756f78b0555a6733ffbf6ab98bd, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:58:14,640 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] ipc.CallRunner(138): callId: 98 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39654 deadline: 1731751154636, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9e28756f78b0555a6733ffbf6ab98bd, server=037c58a19ddb,37267,1731751077844 2024-11-16T09:58:14,641 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731751082172.d9e28756f78b0555a6733ffbf6ab98bd., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 because the exception is null or not the one we care about 2024-11-16T09:58:14,641 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731751082172.d9e28756f78b0555a6733ffbf6ab98bd., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 , the old value is region=TestAcidGuarantees,,1731751082172.d9e28756f78b0555a6733ffbf6ab98bd., hostname=037c58a19ddb,37267,1731751077844, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9e28756f78b0555a6733ffbf6ab98bd, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:58:14,641 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731751082172.d9e28756f78b0555a6733ffbf6ab98bd., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9e28756f78b0555a6733ffbf6ab98bd, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:58:14,641 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731751082172.d9e28756f78b0555a6733ffbf6ab98bd., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 because the exception is null or not the one we care about 2024-11-16T09:58:14,646 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9e28756f78b0555a6733ffbf6ab98bd, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T09:58:14,646 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] ipc.CallRunner(138): callId: 89 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39630 deadline: 1731751154640, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9e28756f78b0555a6733ffbf6ab98bd, server=037c58a19ddb,37267,1731751077844 2024-11-16T09:58:14,646 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731751082172.d9e28756f78b0555a6733ffbf6ab98bd., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 , the old value is region=TestAcidGuarantees,,1731751082172.d9e28756f78b0555a6733ffbf6ab98bd., hostname=037c58a19ddb,37267,1731751077844, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9e28756f78b0555a6733ffbf6ab98bd, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:58:14,646 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731751082172.d9e28756f78b0555a6733ffbf6ab98bd., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9e28756f78b0555a6733ffbf6ab98bd, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:58:14,646 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731751082172.d9e28756f78b0555a6733ffbf6ab98bd., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 because the exception is null or not the one we care about 2024-11-16T09:58:14,646 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9e28756f78b0555a6733ffbf6ab98bd, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T09:58:14,646 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] ipc.CallRunner(138): callId: 98 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39696 deadline: 1731751154640, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9e28756f78b0555a6733ffbf6ab98bd, server=037c58a19ddb,37267,1731751077844 2024-11-16T09:58:14,647 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731751082172.d9e28756f78b0555a6733ffbf6ab98bd., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 , the old value is region=TestAcidGuarantees,,1731751082172.d9e28756f78b0555a6733ffbf6ab98bd., hostname=037c58a19ddb,37267,1731751077844, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9e28756f78b0555a6733ffbf6ab98bd, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:58:14,647 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731751082172.d9e28756f78b0555a6733ffbf6ab98bd., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9e28756f78b0555a6733ffbf6ab98bd, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:58:14,647 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731751082172.d9e28756f78b0555a6733ffbf6ab98bd., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 because the exception is null or not the one we care about 2024-11-16T09:58:14,743 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9e28756f78b0555a6733ffbf6ab98bd, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T09:58:14,743 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] ipc.CallRunner(138): callId: 100 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39620 deadline: 1731751154743, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9e28756f78b0555a6733ffbf6ab98bd, server=037c58a19ddb,37267,1731751077844 2024-11-16T09:58:14,744 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731751082172.d9e28756f78b0555a6733ffbf6ab98bd., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 , the old value is region=TestAcidGuarantees,,1731751082172.d9e28756f78b0555a6733ffbf6ab98bd., hostname=037c58a19ddb,37267,1731751077844, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9e28756f78b0555a6733ffbf6ab98bd, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:58:14,744 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731751082172.d9e28756f78b0555a6733ffbf6ab98bd., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9e28756f78b0555a6733ffbf6ab98bd, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:58:14,744 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731751082172.d9e28756f78b0555a6733ffbf6ab98bd., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 because the exception is null or not the one we care about 2024-11-16T09:58:14,744 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9e28756f78b0555a6733ffbf6ab98bd, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T09:58:14,744 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] ipc.CallRunner(138): callId: 91 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39638 deadline: 1731751154743, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9e28756f78b0555a6733ffbf6ab98bd, server=037c58a19ddb,37267,1731751077844 2024-11-16T09:58:14,744 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9e28756f78b0555a6733ffbf6ab98bd, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T09:58:14,744 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] ipc.CallRunner(138): callId: 99 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39654 deadline: 1731751154743, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9e28756f78b0555a6733ffbf6ab98bd, server=037c58a19ddb,37267,1731751077844 2024-11-16T09:58:14,744 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731751082172.d9e28756f78b0555a6733ffbf6ab98bd., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 , the old value is region=TestAcidGuarantees,,1731751082172.d9e28756f78b0555a6733ffbf6ab98bd., hostname=037c58a19ddb,37267,1731751077844, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9e28756f78b0555a6733ffbf6ab98bd, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:58:14,744 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731751082172.d9e28756f78b0555a6733ffbf6ab98bd., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9e28756f78b0555a6733ffbf6ab98bd, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:58:14,745 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731751082172.d9e28756f78b0555a6733ffbf6ab98bd., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 because the exception is null or not the one we care about 2024-11-16T09:58:14,745 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731751082172.d9e28756f78b0555a6733ffbf6ab98bd., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 , the old value is region=TestAcidGuarantees,,1731751082172.d9e28756f78b0555a6733ffbf6ab98bd., hostname=037c58a19ddb,37267,1731751077844, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9e28756f78b0555a6733ffbf6ab98bd, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:58:14,745 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731751082172.d9e28756f78b0555a6733ffbf6ab98bd., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9e28756f78b0555a6733ffbf6ab98bd, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:58:14,745 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731751082172.d9e28756f78b0555a6733ffbf6ab98bd., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 because the exception is null or not the one we care about 2024-11-16T09:58:14,755 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9e28756f78b0555a6733ffbf6ab98bd, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T09:58:14,756 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9e28756f78b0555a6733ffbf6ab98bd, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T09:58:14,756 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] ipc.CallRunner(138): callId: 90 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39630 deadline: 1731751154753, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9e28756f78b0555a6733ffbf6ab98bd, server=037c58a19ddb,37267,1731751077844 2024-11-16T09:58:14,756 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] ipc.CallRunner(138): callId: 99 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39696 deadline: 1731751154753, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9e28756f78b0555a6733ffbf6ab98bd, server=037c58a19ddb,37267,1731751077844 2024-11-16T09:58:14,756 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731751082172.d9e28756f78b0555a6733ffbf6ab98bd., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 , the old value is region=TestAcidGuarantees,,1731751082172.d9e28756f78b0555a6733ffbf6ab98bd., hostname=037c58a19ddb,37267,1731751077844, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9e28756f78b0555a6733ffbf6ab98bd, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:58:14,756 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731751082172.d9e28756f78b0555a6733ffbf6ab98bd., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9e28756f78b0555a6733ffbf6ab98bd, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:58:14,756 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731751082172.d9e28756f78b0555a6733ffbf6ab98bd., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 , the old value is region=TestAcidGuarantees,,1731751082172.d9e28756f78b0555a6733ffbf6ab98bd., hostname=037c58a19ddb,37267,1731751077844, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9e28756f78b0555a6733ffbf6ab98bd, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:58:14,756 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731751082172.d9e28756f78b0555a6733ffbf6ab98bd., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 because the exception is null or not the one we care about 2024-11-16T09:58:14,756 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731751082172.d9e28756f78b0555a6733ffbf6ab98bd., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9e28756f78b0555a6733ffbf6ab98bd, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:58:14,756 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731751082172.d9e28756f78b0555a6733ffbf6ab98bd., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 because the exception is null or not the one we care about 2024-11-16T09:58:14,955 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9e28756f78b0555a6733ffbf6ab98bd, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T09:58:14,956 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9e28756f78b0555a6733ffbf6ab98bd, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T09:58:14,956 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] ipc.CallRunner(138): callId: 92 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39638 deadline: 1731751154953, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9e28756f78b0555a6733ffbf6ab98bd, server=037c58a19ddb,37267,1731751077844 2024-11-16T09:58:14,956 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] ipc.CallRunner(138): callId: 101 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39620 deadline: 1731751154953, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9e28756f78b0555a6733ffbf6ab98bd, server=037c58a19ddb,37267,1731751077844 2024-11-16T09:58:14,956 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731751082172.d9e28756f78b0555a6733ffbf6ab98bd., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 , the old value is region=TestAcidGuarantees,,1731751082172.d9e28756f78b0555a6733ffbf6ab98bd., hostname=037c58a19ddb,37267,1731751077844, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9e28756f78b0555a6733ffbf6ab98bd, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:58:14,956 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731751082172.d9e28756f78b0555a6733ffbf6ab98bd., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 , the old value is region=TestAcidGuarantees,,1731751082172.d9e28756f78b0555a6733ffbf6ab98bd., hostname=037c58a19ddb,37267,1731751077844, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9e28756f78b0555a6733ffbf6ab98bd, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:58:14,956 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731751082172.d9e28756f78b0555a6733ffbf6ab98bd., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9e28756f78b0555a6733ffbf6ab98bd, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:58:14,956 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731751082172.d9e28756f78b0555a6733ffbf6ab98bd., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9e28756f78b0555a6733ffbf6ab98bd, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:58:14,956 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9e28756f78b0555a6733ffbf6ab98bd, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T09:58:14,956 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731751082172.d9e28756f78b0555a6733ffbf6ab98bd., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 because the exception is null or not the one we care about 2024-11-16T09:58:14,956 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731751082172.d9e28756f78b0555a6733ffbf6ab98bd., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 because the exception is null or not the one we care about 2024-11-16T09:58:14,956 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] ipc.CallRunner(138): callId: 100 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39654 deadline: 1731751154953, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9e28756f78b0555a6733ffbf6ab98bd, server=037c58a19ddb,37267,1731751077844 2024-11-16T09:58:14,957 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731751082172.d9e28756f78b0555a6733ffbf6ab98bd., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 , the old value is region=TestAcidGuarantees,,1731751082172.d9e28756f78b0555a6733ffbf6ab98bd., hostname=037c58a19ddb,37267,1731751077844, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9e28756f78b0555a6733ffbf6ab98bd, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:58:14,957 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731751082172.d9e28756f78b0555a6733ffbf6ab98bd., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9e28756f78b0555a6733ffbf6ab98bd, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:58:14,957 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731751082172.d9e28756f78b0555a6733ffbf6ab98bd., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 because the exception is null or not the one we care about 2024-11-16T09:58:14,965 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9e28756f78b0555a6733ffbf6ab98bd, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T09:58:14,966 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] ipc.CallRunner(138): callId: 100 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39696 deadline: 1731751154963, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9e28756f78b0555a6733ffbf6ab98bd, server=037c58a19ddb,37267,1731751077844 2024-11-16T09:58:14,966 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731751082172.d9e28756f78b0555a6733ffbf6ab98bd., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 , the old value is region=TestAcidGuarantees,,1731751082172.d9e28756f78b0555a6733ffbf6ab98bd., hostname=037c58a19ddb,37267,1731751077844, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9e28756f78b0555a6733ffbf6ab98bd, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:58:14,966 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731751082172.d9e28756f78b0555a6733ffbf6ab98bd., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9e28756f78b0555a6733ffbf6ab98bd, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:58:14,966 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731751082172.d9e28756f78b0555a6733ffbf6ab98bd., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 because the exception is null or not the one we care about 2024-11-16T09:58:14,966 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9e28756f78b0555a6733ffbf6ab98bd, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T09:58:14,966 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] ipc.CallRunner(138): callId: 91 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39630 deadline: 1731751154963, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9e28756f78b0555a6733ffbf6ab98bd, server=037c58a19ddb,37267,1731751077844 2024-11-16T09:58:14,967 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731751082172.d9e28756f78b0555a6733ffbf6ab98bd., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 , the old value is region=TestAcidGuarantees,,1731751082172.d9e28756f78b0555a6733ffbf6ab98bd., hostname=037c58a19ddb,37267,1731751077844, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9e28756f78b0555a6733ffbf6ab98bd, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:58:14,967 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731751082172.d9e28756f78b0555a6733ffbf6ab98bd., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9e28756f78b0555a6733ffbf6ab98bd, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:58:14,967 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731751082172.d9e28756f78b0555a6733ffbf6ab98bd., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 because the exception is null or not the one we care about 2024-11-16T09:58:15,030 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=336 (bloomFilter=true), to=hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/d9e28756f78b0555a6733ffbf6ab98bd/.tmp/A/18344b8f21c345efa4b177f11720008e 2024-11-16T09:58:15,052 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/d9e28756f78b0555a6733ffbf6ab98bd/.tmp/B/bd5c80c59d6c481191c37eb5ef911443 is 50, key is test_row_0/B:col10/1731751094615/Put/seqid=0 2024-11-16T09:58:15,069 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37591 is added to blk_1073741904_1080 (size=12301) 2024-11-16T09:58:15,265 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9e28756f78b0555a6733ffbf6ab98bd, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T09:58:15,265 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] ipc.CallRunner(138): callId: 102 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39620 deadline: 1731751155263, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9e28756f78b0555a6733ffbf6ab98bd, server=037c58a19ddb,37267,1731751077844 2024-11-16T09:58:15,265 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9e28756f78b0555a6733ffbf6ab98bd, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T09:58:15,266 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] ipc.CallRunner(138): callId: 93 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39638 deadline: 1731751155263, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9e28756f78b0555a6733ffbf6ab98bd, server=037c58a19ddb,37267,1731751077844 2024-11-16T09:58:15,266 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731751082172.d9e28756f78b0555a6733ffbf6ab98bd., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 , the old value is region=TestAcidGuarantees,,1731751082172.d9e28756f78b0555a6733ffbf6ab98bd., hostname=037c58a19ddb,37267,1731751077844, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9e28756f78b0555a6733ffbf6ab98bd, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:58:15,266 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9e28756f78b0555a6733ffbf6ab98bd, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T09:58:15,266 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731751082172.d9e28756f78b0555a6733ffbf6ab98bd., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9e28756f78b0555a6733ffbf6ab98bd, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:58:15,266 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731751082172.d9e28756f78b0555a6733ffbf6ab98bd., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 because the exception is null or not the one we care about 2024-11-16T09:58:15,266 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] ipc.CallRunner(138): callId: 101 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39654 deadline: 1731751155263, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9e28756f78b0555a6733ffbf6ab98bd, server=037c58a19ddb,37267,1731751077844 2024-11-16T09:58:15,266 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731751082172.d9e28756f78b0555a6733ffbf6ab98bd., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 , the old value is region=TestAcidGuarantees,,1731751082172.d9e28756f78b0555a6733ffbf6ab98bd., hostname=037c58a19ddb,37267,1731751077844, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9e28756f78b0555a6733ffbf6ab98bd, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:58:15,266 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731751082172.d9e28756f78b0555a6733ffbf6ab98bd., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9e28756f78b0555a6733ffbf6ab98bd, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:58:15,266 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731751082172.d9e28756f78b0555a6733ffbf6ab98bd., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 because the exception is null or not the one we care about 2024-11-16T09:58:15,266 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731751082172.d9e28756f78b0555a6733ffbf6ab98bd., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 , the old value is region=TestAcidGuarantees,,1731751082172.d9e28756f78b0555a6733ffbf6ab98bd., hostname=037c58a19ddb,37267,1731751077844, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9e28756f78b0555a6733ffbf6ab98bd, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:58:15,267 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731751082172.d9e28756f78b0555a6733ffbf6ab98bd., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9e28756f78b0555a6733ffbf6ab98bd, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:58:15,267 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731751082172.d9e28756f78b0555a6733ffbf6ab98bd., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 because the exception is null or not the one we care about 2024-11-16T09:58:15,276 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9e28756f78b0555a6733ffbf6ab98bd, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T09:58:15,276 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] ipc.CallRunner(138): callId: 101 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39696 deadline: 1731751155273, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9e28756f78b0555a6733ffbf6ab98bd, server=037c58a19ddb,37267,1731751077844 2024-11-16T09:58:15,276 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731751082172.d9e28756f78b0555a6733ffbf6ab98bd., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 , the old value is region=TestAcidGuarantees,,1731751082172.d9e28756f78b0555a6733ffbf6ab98bd., hostname=037c58a19ddb,37267,1731751077844, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9e28756f78b0555a6733ffbf6ab98bd, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:58:15,277 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731751082172.d9e28756f78b0555a6733ffbf6ab98bd., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9e28756f78b0555a6733ffbf6ab98bd, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:58:15,277 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731751082172.d9e28756f78b0555a6733ffbf6ab98bd., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 because the exception is null or not the one we care about 2024-11-16T09:58:15,276 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9e28756f78b0555a6733ffbf6ab98bd, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T09:58:15,277 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] ipc.CallRunner(138): callId: 92 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39630 deadline: 1731751155273, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9e28756f78b0555a6733ffbf6ab98bd, server=037c58a19ddb,37267,1731751077844 2024-11-16T09:58:15,277 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731751082172.d9e28756f78b0555a6733ffbf6ab98bd., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 , the old value is region=TestAcidGuarantees,,1731751082172.d9e28756f78b0555a6733ffbf6ab98bd., hostname=037c58a19ddb,37267,1731751077844, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9e28756f78b0555a6733ffbf6ab98bd, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:58:15,277 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731751082172.d9e28756f78b0555a6733ffbf6ab98bd., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9e28756f78b0555a6733ffbf6ab98bd, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:58:15,277 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731751082172.d9e28756f78b0555a6733ffbf6ab98bd., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 because the exception is null or not the one we care about 2024-11-16T09:58:15,470 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=336 (bloomFilter=true), to=hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/d9e28756f78b0555a6733ffbf6ab98bd/.tmp/B/bd5c80c59d6c481191c37eb5ef911443 2024-11-16T09:58:15,481 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/d9e28756f78b0555a6733ffbf6ab98bd/.tmp/C/be1fe7596ca44a539b964b891050ec6f is 50, key is test_row_0/C:col10/1731751094615/Put/seqid=0 2024-11-16T09:58:15,490 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37591 is added to blk_1073741905_1081 (size=12301) 2024-11-16T09:58:15,491 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=336 (bloomFilter=true), to=hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/d9e28756f78b0555a6733ffbf6ab98bd/.tmp/C/be1fe7596ca44a539b964b891050ec6f 2024-11-16T09:58:15,500 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/d9e28756f78b0555a6733ffbf6ab98bd/.tmp/A/18344b8f21c345efa4b177f11720008e as hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/d9e28756f78b0555a6733ffbf6ab98bd/A/18344b8f21c345efa4b177f11720008e 2024-11-16T09:58:15,512 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/d9e28756f78b0555a6733ffbf6ab98bd/A/18344b8f21c345efa4b177f11720008e, entries=250, sequenceid=336, filesize=16.8 K 2024-11-16T09:58:15,516 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/d9e28756f78b0555a6733ffbf6ab98bd/.tmp/B/bd5c80c59d6c481191c37eb5ef911443 as hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/d9e28756f78b0555a6733ffbf6ab98bd/B/bd5c80c59d6c481191c37eb5ef911443 2024-11-16T09:58:15,522 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/d9e28756f78b0555a6733ffbf6ab98bd/B/bd5c80c59d6c481191c37eb5ef911443, entries=150, sequenceid=336, filesize=12.0 K 2024-11-16T09:58:15,524 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/d9e28756f78b0555a6733ffbf6ab98bd/.tmp/C/be1fe7596ca44a539b964b891050ec6f as hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/d9e28756f78b0555a6733ffbf6ab98bd/C/be1fe7596ca44a539b964b891050ec6f 2024-11-16T09:58:15,529 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/d9e28756f78b0555a6733ffbf6ab98bd/C/be1fe7596ca44a539b964b891050ec6f, entries=150, sequenceid=336, filesize=12.0 K 2024-11-16T09:58:15,530 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~147.60 KB/151140, heapSize ~387.42 KB/396720, currentSize=53.67 KB/54960 for d9e28756f78b0555a6733ffbf6ab98bd in 913ms, sequenceid=336, compaction requested=true 2024-11-16T09:58:15,530 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for d9e28756f78b0555a6733ffbf6ab98bd: 2024-11-16T09:58:15,530 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store d9e28756f78b0555a6733ffbf6ab98bd:A, priority=-2147483648, current under compaction store size is 1 2024-11-16T09:58:15,531 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-16T09:58:15,531 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store d9e28756f78b0555a6733ffbf6ab98bd:B, priority=-2147483648, current under compaction store size is 2 2024-11-16T09:58:15,531 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-16T09:58:15,531 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store d9e28756f78b0555a6733ffbf6ab98bd:C, priority=-2147483648, current under compaction store size is 3 2024-11-16T09:58:15,531 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-16T09:58:15,531 DEBUG [RS:0;037c58a19ddb:37267-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-16T09:58:15,531 DEBUG [RS:0;037c58a19ddb:37267-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-16T09:58:15,533 DEBUG [RS:0;037c58a19ddb:37267-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 54732 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-16T09:58:15,533 DEBUG [RS:0;037c58a19ddb:37267-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 49852 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-16T09:58:15,534 DEBUG [RS:0;037c58a19ddb:37267-shortCompactions-0 {}] regionserver.HStore(1541): d9e28756f78b0555a6733ffbf6ab98bd/A is initiating minor compaction (all files) 2024-11-16T09:58:15,534 DEBUG [RS:0;037c58a19ddb:37267-longCompactions-0 {}] regionserver.HStore(1541): d9e28756f78b0555a6733ffbf6ab98bd/B is initiating minor compaction (all files) 2024-11-16T09:58:15,534 INFO [RS:0;037c58a19ddb:37267-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of d9e28756f78b0555a6733ffbf6ab98bd/A in TestAcidGuarantees,,1731751082172.d9e28756f78b0555a6733ffbf6ab98bd. 2024-11-16T09:58:15,534 INFO [RS:0;037c58a19ddb:37267-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of d9e28756f78b0555a6733ffbf6ab98bd/B in TestAcidGuarantees,,1731751082172.d9e28756f78b0555a6733ffbf6ab98bd. 2024-11-16T09:58:15,534 INFO [RS:0;037c58a19ddb:37267-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/d9e28756f78b0555a6733ffbf6ab98bd/A/d737eb9df45049199ca0e28c20466aed, hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/d9e28756f78b0555a6733ffbf6ab98bd/A/3a30ee055c414e25a9cecba7e9183dbe, hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/d9e28756f78b0555a6733ffbf6ab98bd/A/f2ad538a347b4ca9bf7afd4e658e5bac, hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/d9e28756f78b0555a6733ffbf6ab98bd/A/18344b8f21c345efa4b177f11720008e] into tmpdir=hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/d9e28756f78b0555a6733ffbf6ab98bd/.tmp, totalSize=53.4 K 2024-11-16T09:58:15,534 INFO [RS:0;037c58a19ddb:37267-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/d9e28756f78b0555a6733ffbf6ab98bd/B/a73b24e65a43405b9f959afc6671a804, hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/d9e28756f78b0555a6733ffbf6ab98bd/B/0d5fa0a41a674039ac8d14c4f2a5924a, hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/d9e28756f78b0555a6733ffbf6ab98bd/B/5981e9c45ed3447bacfe73467c7101e9, hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/d9e28756f78b0555a6733ffbf6ab98bd/B/bd5c80c59d6c481191c37eb5ef911443] into tmpdir=hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/d9e28756f78b0555a6733ffbf6ab98bd/.tmp, totalSize=48.7 K 2024-11-16T09:58:15,534 DEBUG [RS:0;037c58a19ddb:37267-shortCompactions-0 {}] compactions.Compactor(225): Compacting d737eb9df45049199ca0e28c20466aed, keycount=150, bloomtype=ROW, size=12.6 K, encoding=NONE, compression=NONE, seqNum=271, earliestPutTs=1731751091501 2024-11-16T09:58:15,534 DEBUG [RS:0;037c58a19ddb:37267-longCompactions-0 {}] compactions.Compactor(225): Compacting a73b24e65a43405b9f959afc6671a804, keycount=150, bloomtype=ROW, size=12.6 K, encoding=NONE, compression=NONE, seqNum=271, earliestPutTs=1731751091501 2024-11-16T09:58:15,535 DEBUG [RS:0;037c58a19ddb:37267-shortCompactions-0 {}] compactions.Compactor(225): Compacting 3a30ee055c414e25a9cecba7e9183dbe, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=294, earliestPutTs=1731751091654 2024-11-16T09:58:15,535 DEBUG [RS:0;037c58a19ddb:37267-longCompactions-0 {}] compactions.Compactor(225): Compacting 0d5fa0a41a674039ac8d14c4f2a5924a, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=294, earliestPutTs=1731751091654 2024-11-16T09:58:15,536 DEBUG [RS:0;037c58a19ddb:37267-shortCompactions-0 {}] compactions.Compactor(225): Compacting f2ad538a347b4ca9bf7afd4e658e5bac, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=311, earliestPutTs=1731751091791 2024-11-16T09:58:15,536 DEBUG [RS:0;037c58a19ddb:37267-longCompactions-0 {}] compactions.Compactor(225): Compacting 5981e9c45ed3447bacfe73467c7101e9, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=311, earliestPutTs=1731751091791 2024-11-16T09:58:15,537 DEBUG [RS:0;037c58a19ddb:37267-shortCompactions-0 {}] compactions.Compactor(225): Compacting 18344b8f21c345efa4b177f11720008e, keycount=250, bloomtype=ROW, size=16.8 K, encoding=NONE, compression=NONE, seqNum=336, earliestPutTs=1731751093973 2024-11-16T09:58:15,538 DEBUG [RS:0;037c58a19ddb:37267-longCompactions-0 {}] compactions.Compactor(225): Compacting bd5c80c59d6c481191c37eb5ef911443, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=336, earliestPutTs=1731751093973 2024-11-16T09:58:15,563 INFO [RS:0;037c58a19ddb:37267-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): d9e28756f78b0555a6733ffbf6ab98bd#A#compaction#69 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-16T09:58:15,564 DEBUG [RS:0;037c58a19ddb:37267-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/d9e28756f78b0555a6733ffbf6ab98bd/.tmp/A/32643ed1b9394c95a86ca96dcf7364dd is 50, key is test_row_0/A:col10/1731751094615/Put/seqid=0 2024-11-16T09:58:15,566 INFO [RS:0;037c58a19ddb:37267-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): d9e28756f78b0555a6733ffbf6ab98bd#B#compaction#70 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-16T09:58:15,567 DEBUG [RS:0;037c58a19ddb:37267-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/d9e28756f78b0555a6733ffbf6ab98bd/.tmp/B/b468b9f5c55b4a87ad1e6ee8500a6260 is 50, key is test_row_0/B:col10/1731751094615/Put/seqid=0 2024-11-16T09:58:15,572 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37591 is added to blk_1073741906_1082 (size=13085) 2024-11-16T09:58:15,576 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37591 is added to blk_1073741907_1083 (size=13085) 2024-11-16T09:58:15,585 DEBUG [RS:0;037c58a19ddb:37267-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/d9e28756f78b0555a6733ffbf6ab98bd/.tmp/A/32643ed1b9394c95a86ca96dcf7364dd as hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/d9e28756f78b0555a6733ffbf6ab98bd/A/32643ed1b9394c95a86ca96dcf7364dd 2024-11-16T09:58:15,585 DEBUG [RS:0;037c58a19ddb:37267-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/d9e28756f78b0555a6733ffbf6ab98bd/.tmp/B/b468b9f5c55b4a87ad1e6ee8500a6260 as hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/d9e28756f78b0555a6733ffbf6ab98bd/B/b468b9f5c55b4a87ad1e6ee8500a6260 2024-11-16T09:58:15,595 INFO [RS:0;037c58a19ddb:37267-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in d9e28756f78b0555a6733ffbf6ab98bd/A of d9e28756f78b0555a6733ffbf6ab98bd into 32643ed1b9394c95a86ca96dcf7364dd(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-16T09:58:15,595 INFO [RS:0;037c58a19ddb:37267-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in d9e28756f78b0555a6733ffbf6ab98bd/B of d9e28756f78b0555a6733ffbf6ab98bd into b468b9f5c55b4a87ad1e6ee8500a6260(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-16T09:58:15,595 DEBUG [RS:0;037c58a19ddb:37267-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for d9e28756f78b0555a6733ffbf6ab98bd: 2024-11-16T09:58:15,595 DEBUG [RS:0;037c58a19ddb:37267-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for d9e28756f78b0555a6733ffbf6ab98bd: 2024-11-16T09:58:15,595 INFO [RS:0;037c58a19ddb:37267-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731751082172.d9e28756f78b0555a6733ffbf6ab98bd., storeName=d9e28756f78b0555a6733ffbf6ab98bd/A, priority=12, startTime=1731751095530; duration=0sec 2024-11-16T09:58:15,595 INFO [RS:0;037c58a19ddb:37267-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731751082172.d9e28756f78b0555a6733ffbf6ab98bd., storeName=d9e28756f78b0555a6733ffbf6ab98bd/B, priority=12, startTime=1731751095531; duration=0sec 2024-11-16T09:58:15,595 DEBUG [RS:0;037c58a19ddb:37267-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-16T09:58:15,595 DEBUG [RS:0;037c58a19ddb:37267-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: d9e28756f78b0555a6733ffbf6ab98bd:B 2024-11-16T09:58:15,595 DEBUG [RS:0;037c58a19ddb:37267-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-16T09:58:15,596 DEBUG [RS:0;037c58a19ddb:37267-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-16T09:58:15,596 DEBUG [RS:0;037c58a19ddb:37267-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: d9e28756f78b0555a6733ffbf6ab98bd:A 2024-11-16T09:58:15,601 DEBUG [RS:0;037c58a19ddb:37267-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 49852 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-16T09:58:15,601 DEBUG [RS:0;037c58a19ddb:37267-longCompactions-0 {}] regionserver.HStore(1541): d9e28756f78b0555a6733ffbf6ab98bd/C is initiating minor compaction (all files) 2024-11-16T09:58:15,601 INFO [RS:0;037c58a19ddb:37267-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of d9e28756f78b0555a6733ffbf6ab98bd/C in TestAcidGuarantees,,1731751082172.d9e28756f78b0555a6733ffbf6ab98bd. 2024-11-16T09:58:15,601 INFO [RS:0;037c58a19ddb:37267-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/d9e28756f78b0555a6733ffbf6ab98bd/C/36beb8f139a642b6b398f3dff65758fb, hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/d9e28756f78b0555a6733ffbf6ab98bd/C/5058b1fc65bf44d6b262d1e0e572bc03, hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/d9e28756f78b0555a6733ffbf6ab98bd/C/d869628b6caf43adb4766aab62057038, hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/d9e28756f78b0555a6733ffbf6ab98bd/C/be1fe7596ca44a539b964b891050ec6f] into tmpdir=hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/d9e28756f78b0555a6733ffbf6ab98bd/.tmp, totalSize=48.7 K 2024-11-16T09:58:15,601 DEBUG [RS:0;037c58a19ddb:37267-longCompactions-0 {}] compactions.Compactor(225): Compacting 36beb8f139a642b6b398f3dff65758fb, keycount=150, bloomtype=ROW, size=12.6 K, encoding=NONE, compression=NONE, seqNum=271, earliestPutTs=1731751091501 2024-11-16T09:58:15,602 DEBUG [RS:0;037c58a19ddb:37267-longCompactions-0 {}] compactions.Compactor(225): Compacting 5058b1fc65bf44d6b262d1e0e572bc03, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=294, earliestPutTs=1731751091654 2024-11-16T09:58:15,602 DEBUG [RS:0;037c58a19ddb:37267-longCompactions-0 {}] compactions.Compactor(225): Compacting d869628b6caf43adb4766aab62057038, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=311, earliestPutTs=1731751091791 2024-11-16T09:58:15,602 DEBUG [RS:0;037c58a19ddb:37267-longCompactions-0 {}] compactions.Compactor(225): Compacting be1fe7596ca44a539b964b891050ec6f, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=336, earliestPutTs=1731751093973 2024-11-16T09:58:15,614 INFO [RS:0;037c58a19ddb:37267-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): d9e28756f78b0555a6733ffbf6ab98bd#C#compaction#71 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-16T09:58:15,615 DEBUG [RS:0;037c58a19ddb:37267-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/d9e28756f78b0555a6733ffbf6ab98bd/.tmp/C/167e57611134426aa91dd1a849a17aae is 50, key is test_row_0/C:col10/1731751094615/Put/seqid=0 2024-11-16T09:58:15,627 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37591 is added to blk_1073741908_1084 (size=13085) 2024-11-16T09:58:15,639 DEBUG [RS:0;037c58a19ddb:37267-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/d9e28756f78b0555a6733ffbf6ab98bd/.tmp/C/167e57611134426aa91dd1a849a17aae as hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/d9e28756f78b0555a6733ffbf6ab98bd/C/167e57611134426aa91dd1a849a17aae 2024-11-16T09:58:15,649 INFO [RS:0;037c58a19ddb:37267-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in d9e28756f78b0555a6733ffbf6ab98bd/C of d9e28756f78b0555a6733ffbf6ab98bd into 167e57611134426aa91dd1a849a17aae(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-16T09:58:15,649 DEBUG [RS:0;037c58a19ddb:37267-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for d9e28756f78b0555a6733ffbf6ab98bd: 2024-11-16T09:58:15,649 INFO [RS:0;037c58a19ddb:37267-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731751082172.d9e28756f78b0555a6733ffbf6ab98bd., storeName=d9e28756f78b0555a6733ffbf6ab98bd/C, priority=12, startTime=1731751095531; duration=0sec 2024-11-16T09:58:15,649 DEBUG [RS:0;037c58a19ddb:37267-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-16T09:58:15,649 DEBUG [RS:0;037c58a19ddb:37267-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: d9e28756f78b0555a6733ffbf6ab98bd:C 2024-11-16T09:58:15,775 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] regionserver.HRegion(8855): Flush requested on d9e28756f78b0555a6733ffbf6ab98bd 2024-11-16T09:58:15,775 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-16T09:58:15,775 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing d9e28756f78b0555a6733ffbf6ab98bd 3/3 column families, dataSize=73.80 KB heapSize=194.11 KB 2024-11-16T09:58:15,782 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/d9e28756f78b0555a6733ffbf6ab98bd/.tmp/A/d1f88cc47aef44b2a64aed41364c5430 is 50, key is test_row_0/A:col10/1731751094632/Put/seqid=0 2024-11-16T09:58:15,793 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37591 is added to blk_1073741909_1085 (size=12301) 2024-11-16T09:58:15,838 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9e28756f78b0555a6733ffbf6ab98bd, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T09:58:15,839 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] ipc.CallRunner(138): callId: 96 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39630 deadline: 1731751155835, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9e28756f78b0555a6733ffbf6ab98bd, server=037c58a19ddb,37267,1731751077844 2024-11-16T09:58:15,839 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731751082172.d9e28756f78b0555a6733ffbf6ab98bd., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 , the old value is region=TestAcidGuarantees,,1731751082172.d9e28756f78b0555a6733ffbf6ab98bd., hostname=037c58a19ddb,37267,1731751077844, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9e28756f78b0555a6733ffbf6ab98bd, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:58:15,839 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9e28756f78b0555a6733ffbf6ab98bd, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T09:58:15,839 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731751082172.d9e28756f78b0555a6733ffbf6ab98bd., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9e28756f78b0555a6733ffbf6ab98bd, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:58:15,839 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731751082172.d9e28756f78b0555a6733ffbf6ab98bd., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 because the exception is null or not the one we care about 2024-11-16T09:58:15,839 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] ipc.CallRunner(138): callId: 108 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39620 deadline: 1731751155837, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9e28756f78b0555a6733ffbf6ab98bd, server=037c58a19ddb,37267,1731751077844 2024-11-16T09:58:15,840 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731751082172.d9e28756f78b0555a6733ffbf6ab98bd., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 , the old value is region=TestAcidGuarantees,,1731751082172.d9e28756f78b0555a6733ffbf6ab98bd., hostname=037c58a19ddb,37267,1731751077844, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9e28756f78b0555a6733ffbf6ab98bd, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:58:15,840 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731751082172.d9e28756f78b0555a6733ffbf6ab98bd., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9e28756f78b0555a6733ffbf6ab98bd, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:58:15,840 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731751082172.d9e28756f78b0555a6733ffbf6ab98bd., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 because the exception is null or not the one we care about 2024-11-16T09:58:15,841 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9e28756f78b0555a6733ffbf6ab98bd, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T09:58:15,842 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] ipc.CallRunner(138): callId: 108 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39654 deadline: 1731751155837, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9e28756f78b0555a6733ffbf6ab98bd, server=037c58a19ddb,37267,1731751077844 2024-11-16T09:58:15,842 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9e28756f78b0555a6733ffbf6ab98bd, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T09:58:15,842 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731751082172.d9e28756f78b0555a6733ffbf6ab98bd., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 , the old value is region=TestAcidGuarantees,,1731751082172.d9e28756f78b0555a6733ffbf6ab98bd., hostname=037c58a19ddb,37267,1731751077844, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9e28756f78b0555a6733ffbf6ab98bd, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:58:15,842 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731751082172.d9e28756f78b0555a6733ffbf6ab98bd., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9e28756f78b0555a6733ffbf6ab98bd, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:58:15,842 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731751082172.d9e28756f78b0555a6733ffbf6ab98bd., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 because the exception is null or not the one we care about 2024-11-16T09:58:15,842 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] ipc.CallRunner(138): callId: 106 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39696 deadline: 1731751155839, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9e28756f78b0555a6733ffbf6ab98bd, server=037c58a19ddb,37267,1731751077844 2024-11-16T09:58:15,842 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731751082172.d9e28756f78b0555a6733ffbf6ab98bd., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 , the old value is region=TestAcidGuarantees,,1731751082172.d9e28756f78b0555a6733ffbf6ab98bd., hostname=037c58a19ddb,37267,1731751077844, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9e28756f78b0555a6733ffbf6ab98bd, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:58:15,842 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9e28756f78b0555a6733ffbf6ab98bd, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T09:58:15,843 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731751082172.d9e28756f78b0555a6733ffbf6ab98bd., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9e28756f78b0555a6733ffbf6ab98bd, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:58:15,843 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731751082172.d9e28756f78b0555a6733ffbf6ab98bd., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 because the exception is null or not the one we care about 2024-11-16T09:58:15,843 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] ipc.CallRunner(138): callId: 99 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39638 deadline: 1731751155839, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9e28756f78b0555a6733ffbf6ab98bd, server=037c58a19ddb,37267,1731751077844 2024-11-16T09:58:15,843 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731751082172.d9e28756f78b0555a6733ffbf6ab98bd., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 , the old value is region=TestAcidGuarantees,,1731751082172.d9e28756f78b0555a6733ffbf6ab98bd., hostname=037c58a19ddb,37267,1731751077844, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9e28756f78b0555a6733ffbf6ab98bd, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:58:15,843 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731751082172.d9e28756f78b0555a6733ffbf6ab98bd., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9e28756f78b0555a6733ffbf6ab98bd, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:58:15,843 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731751082172.d9e28756f78b0555a6733ffbf6ab98bd., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 because the exception is null or not the one we care about 2024-11-16T09:58:15,945 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9e28756f78b0555a6733ffbf6ab98bd, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T09:58:15,946 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] ipc.CallRunner(138): callId: 97 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39630 deadline: 1731751155943, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9e28756f78b0555a6733ffbf6ab98bd, server=037c58a19ddb,37267,1731751077844 2024-11-16T09:58:15,946 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731751082172.d9e28756f78b0555a6733ffbf6ab98bd., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 , the old value is region=TestAcidGuarantees,,1731751082172.d9e28756f78b0555a6733ffbf6ab98bd., hostname=037c58a19ddb,37267,1731751077844, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9e28756f78b0555a6733ffbf6ab98bd, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:58:15,946 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9e28756f78b0555a6733ffbf6ab98bd, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T09:58:15,946 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731751082172.d9e28756f78b0555a6733ffbf6ab98bd., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9e28756f78b0555a6733ffbf6ab98bd, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:58:15,946 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731751082172.d9e28756f78b0555a6733ffbf6ab98bd., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 because the exception is null or not the one we care about 2024-11-16T09:58:15,946 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] ipc.CallRunner(138): callId: 109 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39620 deadline: 1731751155943, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9e28756f78b0555a6733ffbf6ab98bd, server=037c58a19ddb,37267,1731751077844 2024-11-16T09:58:15,947 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731751082172.d9e28756f78b0555a6733ffbf6ab98bd., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 , the old value is region=TestAcidGuarantees,,1731751082172.d9e28756f78b0555a6733ffbf6ab98bd., hostname=037c58a19ddb,37267,1731751077844, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9e28756f78b0555a6733ffbf6ab98bd, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:58:15,947 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731751082172.d9e28756f78b0555a6733ffbf6ab98bd., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9e28756f78b0555a6733ffbf6ab98bd, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:58:15,947 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731751082172.d9e28756f78b0555a6733ffbf6ab98bd., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 because the exception is null or not the one we care about 2024-11-16T09:58:15,954 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9e28756f78b0555a6733ffbf6ab98bd, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T09:58:15,954 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] ipc.CallRunner(138): callId: 109 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39654 deadline: 1731751155952, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9e28756f78b0555a6733ffbf6ab98bd, server=037c58a19ddb,37267,1731751077844 2024-11-16T09:58:15,955 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731751082172.d9e28756f78b0555a6733ffbf6ab98bd., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 , the old value is region=TestAcidGuarantees,,1731751082172.d9e28756f78b0555a6733ffbf6ab98bd., hostname=037c58a19ddb,37267,1731751077844, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9e28756f78b0555a6733ffbf6ab98bd, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:58:15,955 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731751082172.d9e28756f78b0555a6733ffbf6ab98bd., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9e28756f78b0555a6733ffbf6ab98bd, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:58:15,955 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9e28756f78b0555a6733ffbf6ab98bd, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T09:58:15,955 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731751082172.d9e28756f78b0555a6733ffbf6ab98bd., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 because the exception is null or not the one we care about 2024-11-16T09:58:15,955 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] ipc.CallRunner(138): callId: 107 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39696 deadline: 1731751155953, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9e28756f78b0555a6733ffbf6ab98bd, server=037c58a19ddb,37267,1731751077844 2024-11-16T09:58:15,955 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731751082172.d9e28756f78b0555a6733ffbf6ab98bd., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 , the old value is region=TestAcidGuarantees,,1731751082172.d9e28756f78b0555a6733ffbf6ab98bd., hostname=037c58a19ddb,37267,1731751077844, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9e28756f78b0555a6733ffbf6ab98bd, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:58:15,955 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9e28756f78b0555a6733ffbf6ab98bd, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T09:58:15,955 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731751082172.d9e28756f78b0555a6733ffbf6ab98bd., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9e28756f78b0555a6733ffbf6ab98bd, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:58:15,955 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731751082172.d9e28756f78b0555a6733ffbf6ab98bd., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 because the exception is null or not the one we care about 2024-11-16T09:58:15,955 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] ipc.CallRunner(138): callId: 100 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39638 deadline: 1731751155953, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9e28756f78b0555a6733ffbf6ab98bd, server=037c58a19ddb,37267,1731751077844 2024-11-16T09:58:15,956 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731751082172.d9e28756f78b0555a6733ffbf6ab98bd., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 , the old value is region=TestAcidGuarantees,,1731751082172.d9e28756f78b0555a6733ffbf6ab98bd., hostname=037c58a19ddb,37267,1731751077844, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9e28756f78b0555a6733ffbf6ab98bd, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:58:15,956 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731751082172.d9e28756f78b0555a6733ffbf6ab98bd., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9e28756f78b0555a6733ffbf6ab98bd, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:58:15,956 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731751082172.d9e28756f78b0555a6733ffbf6ab98bd., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 because the exception is null or not the one we care about 2024-11-16T09:58:16,093 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37877 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=13 2024-11-16T09:58:16,093 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-16T09:58:16,094 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37877 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-16T09:58:16,095 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37877 {}] procedure2.ProcedureExecutor(1139): Stored pid=15, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=15, table=TestAcidGuarantees 2024-11-16T09:58:16,096 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=15, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=15, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-16T09:58:16,096 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37877 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=15 2024-11-16T09:58:16,097 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=15, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=15, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-16T09:58:16,097 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=16, ppid=15, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-16T09:58:16,155 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9e28756f78b0555a6733ffbf6ab98bd, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T09:58:16,155 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] ipc.CallRunner(138): callId: 98 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39630 deadline: 1731751156152, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9e28756f78b0555a6733ffbf6ab98bd, server=037c58a19ddb,37267,1731751077844 2024-11-16T09:58:16,156 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731751082172.d9e28756f78b0555a6733ffbf6ab98bd., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 , the old value is region=TestAcidGuarantees,,1731751082172.d9e28756f78b0555a6733ffbf6ab98bd., hostname=037c58a19ddb,37267,1731751077844, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9e28756f78b0555a6733ffbf6ab98bd, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:58:16,156 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731751082172.d9e28756f78b0555a6733ffbf6ab98bd., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9e28756f78b0555a6733ffbf6ab98bd, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:58:16,156 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731751082172.d9e28756f78b0555a6733ffbf6ab98bd., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 because the exception is null or not the one we care about 2024-11-16T09:58:16,156 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9e28756f78b0555a6733ffbf6ab98bd, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T09:58:16,156 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] ipc.CallRunner(138): callId: 110 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39620 deadline: 1731751156153, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9e28756f78b0555a6733ffbf6ab98bd, server=037c58a19ddb,37267,1731751077844 2024-11-16T09:58:16,157 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731751082172.d9e28756f78b0555a6733ffbf6ab98bd., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 , the old value is region=TestAcidGuarantees,,1731751082172.d9e28756f78b0555a6733ffbf6ab98bd., hostname=037c58a19ddb,37267,1731751077844, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9e28756f78b0555a6733ffbf6ab98bd, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:58:16,157 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731751082172.d9e28756f78b0555a6733ffbf6ab98bd., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9e28756f78b0555a6733ffbf6ab98bd, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:58:16,157 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731751082172.d9e28756f78b0555a6733ffbf6ab98bd., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 because the exception is null or not the one we care about 2024-11-16T09:58:16,167 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9e28756f78b0555a6733ffbf6ab98bd, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T09:58:16,167 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] ipc.CallRunner(138): callId: 110 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39654 deadline: 1731751156163, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9e28756f78b0555a6733ffbf6ab98bd, server=037c58a19ddb,37267,1731751077844 2024-11-16T09:58:16,168 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731751082172.d9e28756f78b0555a6733ffbf6ab98bd., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 , the old value is region=TestAcidGuarantees,,1731751082172.d9e28756f78b0555a6733ffbf6ab98bd., hostname=037c58a19ddb,37267,1731751077844, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9e28756f78b0555a6733ffbf6ab98bd, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:58:16,168 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731751082172.d9e28756f78b0555a6733ffbf6ab98bd., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9e28756f78b0555a6733ffbf6ab98bd, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:58:16,168 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9e28756f78b0555a6733ffbf6ab98bd, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T09:58:16,168 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731751082172.d9e28756f78b0555a6733ffbf6ab98bd., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 because the exception is null or not the one we care about 2024-11-16T09:58:16,168 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] ipc.CallRunner(138): callId: 108 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39696 deadline: 1731751156163, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9e28756f78b0555a6733ffbf6ab98bd, server=037c58a19ddb,37267,1731751077844 2024-11-16T09:58:16,168 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731751082172.d9e28756f78b0555a6733ffbf6ab98bd., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 , the old value is region=TestAcidGuarantees,,1731751082172.d9e28756f78b0555a6733ffbf6ab98bd., hostname=037c58a19ddb,37267,1731751077844, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9e28756f78b0555a6733ffbf6ab98bd, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:58:16,168 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731751082172.d9e28756f78b0555a6733ffbf6ab98bd., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9e28756f78b0555a6733ffbf6ab98bd, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:58:16,168 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731751082172.d9e28756f78b0555a6733ffbf6ab98bd., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 because the exception is null or not the one we care about 2024-11-16T09:58:16,168 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9e28756f78b0555a6733ffbf6ab98bd, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T09:58:16,168 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] ipc.CallRunner(138): callId: 101 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39638 deadline: 1731751156163, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9e28756f78b0555a6733ffbf6ab98bd, server=037c58a19ddb,37267,1731751077844 2024-11-16T09:58:16,169 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731751082172.d9e28756f78b0555a6733ffbf6ab98bd., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 , the old value is region=TestAcidGuarantees,,1731751082172.d9e28756f78b0555a6733ffbf6ab98bd., hostname=037c58a19ddb,37267,1731751077844, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9e28756f78b0555a6733ffbf6ab98bd, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:58:16,169 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731751082172.d9e28756f78b0555a6733ffbf6ab98bd., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9e28756f78b0555a6733ffbf6ab98bd, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:58:16,169 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731751082172.d9e28756f78b0555a6733ffbf6ab98bd., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 because the exception is null or not the one we care about 2024-11-16T09:58:16,195 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=353 (bloomFilter=true), to=hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/d9e28756f78b0555a6733ffbf6ab98bd/.tmp/A/d1f88cc47aef44b2a64aed41364c5430 2024-11-16T09:58:16,203 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37877 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=15 2024-11-16T09:58:16,207 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/d9e28756f78b0555a6733ffbf6ab98bd/.tmp/B/3fade5830bae4cc28018c7702bc3b811 is 50, key is test_row_0/B:col10/1731751094632/Put/seqid=0 2024-11-16T09:58:16,213 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37591 is added to blk_1073741910_1086 (size=12301) 2024-11-16T09:58:16,252 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=37267 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=16 2024-11-16T09:58:16,252 DEBUG [RS_FLUSH_OPERATIONS-regionserver/037c58a19ddb:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731751082172.d9e28756f78b0555a6733ffbf6ab98bd. 2024-11-16T09:58:16,253 DEBUG [RS_FLUSH_OPERATIONS-regionserver/037c58a19ddb:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731751082172.d9e28756f78b0555a6733ffbf6ab98bd. as already flushing 2024-11-16T09:58:16,253 DEBUG [RS_FLUSH_OPERATIONS-regionserver/037c58a19ddb:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731751082172.d9e28756f78b0555a6733ffbf6ab98bd. 2024-11-16T09:58:16,253 ERROR [RS_FLUSH_OPERATIONS-regionserver/037c58a19ddb:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] handler.RSProcedureHandler(58): pid=16 java.io.IOException: Unable to complete flush {ENCODED => d9e28756f78b0555a6733ffbf6ab98bd, NAME => 'TestAcidGuarantees,,1731751082172.d9e28756f78b0555a6733ffbf6ab98bd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T09:58:16,253 DEBUG [RS_FLUSH_OPERATIONS-regionserver/037c58a19ddb:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=16 java.io.IOException: Unable to complete flush {ENCODED => d9e28756f78b0555a6733ffbf6ab98bd, NAME => 'TestAcidGuarantees,,1731751082172.d9e28756f78b0555a6733ffbf6ab98bd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T09:58:16,254 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37877 {}] master.HMaster(4177): Remote procedure failed, pid=16 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d9e28756f78b0555a6733ffbf6ab98bd, NAME => 'TestAcidGuarantees,,1731751082172.d9e28756f78b0555a6733ffbf6ab98bd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d9e28756f78b0555a6733ffbf6ab98bd, NAME => 'TestAcidGuarantees,,1731751082172.d9e28756f78b0555a6733ffbf6ab98bd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T09:58:16,406 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=37267 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=16 2024-11-16T09:58:16,406 DEBUG [RS_FLUSH_OPERATIONS-regionserver/037c58a19ddb:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731751082172.d9e28756f78b0555a6733ffbf6ab98bd. 2024-11-16T09:58:16,407 DEBUG [RS_FLUSH_OPERATIONS-regionserver/037c58a19ddb:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731751082172.d9e28756f78b0555a6733ffbf6ab98bd. as already flushing 2024-11-16T09:58:16,407 DEBUG [RS_FLUSH_OPERATIONS-regionserver/037c58a19ddb:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731751082172.d9e28756f78b0555a6733ffbf6ab98bd. 2024-11-16T09:58:16,407 ERROR [RS_FLUSH_OPERATIONS-regionserver/037c58a19ddb:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] handler.RSProcedureHandler(58): pid=16 java.io.IOException: Unable to complete flush {ENCODED => d9e28756f78b0555a6733ffbf6ab98bd, NAME => 'TestAcidGuarantees,,1731751082172.d9e28756f78b0555a6733ffbf6ab98bd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T09:58:16,407 DEBUG [RS_FLUSH_OPERATIONS-regionserver/037c58a19ddb:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=16 java.io.IOException: Unable to complete flush {ENCODED => d9e28756f78b0555a6733ffbf6ab98bd, NAME => 'TestAcidGuarantees,,1731751082172.d9e28756f78b0555a6733ffbf6ab98bd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T09:58:16,408 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37877 {}] master.HMaster(4177): Remote procedure failed, pid=16 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d9e28756f78b0555a6733ffbf6ab98bd, NAME => 'TestAcidGuarantees,,1731751082172.d9e28756f78b0555a6733ffbf6ab98bd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d9e28756f78b0555a6733ffbf6ab98bd, NAME => 'TestAcidGuarantees,,1731751082172.d9e28756f78b0555a6733ffbf6ab98bd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T09:58:16,413 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37877 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=15 2024-11-16T09:58:16,468 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9e28756f78b0555a6733ffbf6ab98bd, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T09:58:16,468 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9e28756f78b0555a6733ffbf6ab98bd, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T09:58:16,468 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] ipc.CallRunner(138): callId: 111 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39620 deadline: 1731751156463, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9e28756f78b0555a6733ffbf6ab98bd, server=037c58a19ddb,37267,1731751077844 2024-11-16T09:58:16,468 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] ipc.CallRunner(138): callId: 99 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39630 deadline: 1731751156463, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9e28756f78b0555a6733ffbf6ab98bd, server=037c58a19ddb,37267,1731751077844 2024-11-16T09:58:16,469 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731751082172.d9e28756f78b0555a6733ffbf6ab98bd., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 , the old value is region=TestAcidGuarantees,,1731751082172.d9e28756f78b0555a6733ffbf6ab98bd., hostname=037c58a19ddb,37267,1731751077844, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9e28756f78b0555a6733ffbf6ab98bd, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:58:16,469 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731751082172.d9e28756f78b0555a6733ffbf6ab98bd., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9e28756f78b0555a6733ffbf6ab98bd, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:58:16,469 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731751082172.d9e28756f78b0555a6733ffbf6ab98bd., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 , the old value is region=TestAcidGuarantees,,1731751082172.d9e28756f78b0555a6733ffbf6ab98bd., hostname=037c58a19ddb,37267,1731751077844, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9e28756f78b0555a6733ffbf6ab98bd, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:58:16,469 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731751082172.d9e28756f78b0555a6733ffbf6ab98bd., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 because the exception is null or not the one we care about 2024-11-16T09:58:16,469 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731751082172.d9e28756f78b0555a6733ffbf6ab98bd., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9e28756f78b0555a6733ffbf6ab98bd, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:58:16,469 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731751082172.d9e28756f78b0555a6733ffbf6ab98bd., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 because the exception is null or not the one we care about 2024-11-16T09:58:16,476 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9e28756f78b0555a6733ffbf6ab98bd, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T09:58:16,476 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] ipc.CallRunner(138): callId: 111 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39654 deadline: 1731751156473, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9e28756f78b0555a6733ffbf6ab98bd, server=037c58a19ddb,37267,1731751077844 2024-11-16T09:58:16,476 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731751082172.d9e28756f78b0555a6733ffbf6ab98bd., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 , the old value is region=TestAcidGuarantees,,1731751082172.d9e28756f78b0555a6733ffbf6ab98bd., hostname=037c58a19ddb,37267,1731751077844, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9e28756f78b0555a6733ffbf6ab98bd, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:58:16,476 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731751082172.d9e28756f78b0555a6733ffbf6ab98bd., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9e28756f78b0555a6733ffbf6ab98bd, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:58:16,477 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731751082172.d9e28756f78b0555a6733ffbf6ab98bd., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 because the exception is null or not the one we care about 2024-11-16T09:58:16,476 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9e28756f78b0555a6733ffbf6ab98bd, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T09:58:16,477 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] ipc.CallRunner(138): callId: 109 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39696 deadline: 1731751156473, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9e28756f78b0555a6733ffbf6ab98bd, server=037c58a19ddb,37267,1731751077844 2024-11-16T09:58:16,477 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731751082172.d9e28756f78b0555a6733ffbf6ab98bd., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 , the old value is region=TestAcidGuarantees,,1731751082172.d9e28756f78b0555a6733ffbf6ab98bd., hostname=037c58a19ddb,37267,1731751077844, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9e28756f78b0555a6733ffbf6ab98bd, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:58:16,477 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731751082172.d9e28756f78b0555a6733ffbf6ab98bd., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9e28756f78b0555a6733ffbf6ab98bd, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:58:16,477 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731751082172.d9e28756f78b0555a6733ffbf6ab98bd., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 because the exception is null or not the one we care about 2024-11-16T09:58:16,477 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9e28756f78b0555a6733ffbf6ab98bd, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T09:58:16,477 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] ipc.CallRunner(138): callId: 102 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39638 deadline: 1731751156473, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9e28756f78b0555a6733ffbf6ab98bd, server=037c58a19ddb,37267,1731751077844 2024-11-16T09:58:16,477 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731751082172.d9e28756f78b0555a6733ffbf6ab98bd., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 , the old value is region=TestAcidGuarantees,,1731751082172.d9e28756f78b0555a6733ffbf6ab98bd., hostname=037c58a19ddb,37267,1731751077844, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9e28756f78b0555a6733ffbf6ab98bd, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:58:16,478 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731751082172.d9e28756f78b0555a6733ffbf6ab98bd., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9e28756f78b0555a6733ffbf6ab98bd, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:58:16,478 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731751082172.d9e28756f78b0555a6733ffbf6ab98bd., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 because the exception is null or not the one we care about 2024-11-16T09:58:16,567 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=37267 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=16 2024-11-16T09:58:16,567 DEBUG [RS_FLUSH_OPERATIONS-regionserver/037c58a19ddb:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731751082172.d9e28756f78b0555a6733ffbf6ab98bd. 2024-11-16T09:58:16,567 DEBUG [RS_FLUSH_OPERATIONS-regionserver/037c58a19ddb:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731751082172.d9e28756f78b0555a6733ffbf6ab98bd. as already flushing 2024-11-16T09:58:16,567 DEBUG [RS_FLUSH_OPERATIONS-regionserver/037c58a19ddb:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731751082172.d9e28756f78b0555a6733ffbf6ab98bd. 2024-11-16T09:58:16,567 ERROR [RS_FLUSH_OPERATIONS-regionserver/037c58a19ddb:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] handler.RSProcedureHandler(58): pid=16 java.io.IOException: Unable to complete flush {ENCODED => d9e28756f78b0555a6733ffbf6ab98bd, NAME => 'TestAcidGuarantees,,1731751082172.d9e28756f78b0555a6733ffbf6ab98bd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T09:58:16,567 DEBUG [RS_FLUSH_OPERATIONS-regionserver/037c58a19ddb:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=16 java.io.IOException: Unable to complete flush {ENCODED => d9e28756f78b0555a6733ffbf6ab98bd, NAME => 'TestAcidGuarantees,,1731751082172.d9e28756f78b0555a6733ffbf6ab98bd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T09:58:16,568 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37877 {}] master.HMaster(4177): Remote procedure failed, pid=16 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d9e28756f78b0555a6733ffbf6ab98bd, NAME => 'TestAcidGuarantees,,1731751082172.d9e28756f78b0555a6733ffbf6ab98bd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d9e28756f78b0555a6733ffbf6ab98bd, NAME => 'TestAcidGuarantees,,1731751082172.d9e28756f78b0555a6733ffbf6ab98bd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T09:58:16,614 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=353 (bloomFilter=true), to=hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/d9e28756f78b0555a6733ffbf6ab98bd/.tmp/B/3fade5830bae4cc28018c7702bc3b811 2024-11-16T09:58:16,624 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/d9e28756f78b0555a6733ffbf6ab98bd/.tmp/C/0e51d1c5bd2e4038af191f3febcb368a is 50, key is test_row_0/C:col10/1731751094632/Put/seqid=0 2024-11-16T09:58:16,628 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37591 is added to blk_1073741911_1087 (size=12301) 2024-11-16T09:58:16,630 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=353 (bloomFilter=true), to=hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/d9e28756f78b0555a6733ffbf6ab98bd/.tmp/C/0e51d1c5bd2e4038af191f3febcb368a 2024-11-16T09:58:16,636 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/d9e28756f78b0555a6733ffbf6ab98bd/.tmp/A/d1f88cc47aef44b2a64aed41364c5430 as hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/d9e28756f78b0555a6733ffbf6ab98bd/A/d1f88cc47aef44b2a64aed41364c5430 2024-11-16T09:58:16,641 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/d9e28756f78b0555a6733ffbf6ab98bd/A/d1f88cc47aef44b2a64aed41364c5430, entries=150, sequenceid=353, filesize=12.0 K 2024-11-16T09:58:16,642 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/d9e28756f78b0555a6733ffbf6ab98bd/.tmp/B/3fade5830bae4cc28018c7702bc3b811 as hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/d9e28756f78b0555a6733ffbf6ab98bd/B/3fade5830bae4cc28018c7702bc3b811 2024-11-16T09:58:16,653 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/d9e28756f78b0555a6733ffbf6ab98bd/B/3fade5830bae4cc28018c7702bc3b811, entries=150, sequenceid=353, filesize=12.0 K 2024-11-16T09:58:16,655 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/d9e28756f78b0555a6733ffbf6ab98bd/.tmp/C/0e51d1c5bd2e4038af191f3febcb368a as hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/d9e28756f78b0555a6733ffbf6ab98bd/C/0e51d1c5bd2e4038af191f3febcb368a 2024-11-16T09:58:16,660 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/d9e28756f78b0555a6733ffbf6ab98bd/C/0e51d1c5bd2e4038af191f3febcb368a, entries=150, sequenceid=353, filesize=12.0 K 2024-11-16T09:58:16,662 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~73.80 KB/75570, heapSize ~194.06 KB/198720, currentSize=134.18 KB/137400 for d9e28756f78b0555a6733ffbf6ab98bd in 887ms, sequenceid=353, compaction requested=false 2024-11-16T09:58:16,662 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for d9e28756f78b0555a6733ffbf6ab98bd: 2024-11-16T09:58:16,721 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=37267 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=16 2024-11-16T09:58:16,721 DEBUG [RS_FLUSH_OPERATIONS-regionserver/037c58a19ddb:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731751082172.d9e28756f78b0555a6733ffbf6ab98bd. 2024-11-16T09:58:16,721 INFO [RS_FLUSH_OPERATIONS-regionserver/037c58a19ddb:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegion(2902): Flushing d9e28756f78b0555a6733ffbf6ab98bd 3/3 column families, dataSize=134.18 KB heapSize=352.31 KB 2024-11-16T09:58:16,725 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37877 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=15 2024-11-16T09:58:16,728 DEBUG [RS_FLUSH_OPERATIONS-regionserver/037c58a19ddb:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/d9e28756f78b0555a6733ffbf6ab98bd/.tmp/A/193f043412fd40d3989d5e43a5329301 is 50, key is test_row_0/A:col10/1731751095827/Put/seqid=0 2024-11-16T09:58:16,738 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37591 is added to blk_1073741912_1088 (size=12301) 2024-11-16T09:58:16,739 INFO [RS_FLUSH_OPERATIONS-regionserver/037c58a19ddb:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=376 (bloomFilter=true), to=hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/d9e28756f78b0555a6733ffbf6ab98bd/.tmp/A/193f043412fd40d3989d5e43a5329301 2024-11-16T09:58:16,751 DEBUG [RS_FLUSH_OPERATIONS-regionserver/037c58a19ddb:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/d9e28756f78b0555a6733ffbf6ab98bd/.tmp/B/948594f9b3c641f28c66b71ad0b18386 is 50, key is test_row_0/B:col10/1731751095827/Put/seqid=0 2024-11-16T09:58:16,759 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37591 is added to blk_1073741913_1089 (size=12301) 2024-11-16T09:58:16,760 INFO [RS_FLUSH_OPERATIONS-regionserver/037c58a19ddb:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=376 (bloomFilter=true), to=hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/d9e28756f78b0555a6733ffbf6ab98bd/.tmp/B/948594f9b3c641f28c66b71ad0b18386 2024-11-16T09:58:16,773 DEBUG [RS_FLUSH_OPERATIONS-regionserver/037c58a19ddb:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/d9e28756f78b0555a6733ffbf6ab98bd/.tmp/C/98262bb2b6fb41ecbf6e44ee204fa711 is 50, key is test_row_0/C:col10/1731751095827/Put/seqid=0 2024-11-16T09:58:16,786 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37591 is added to blk_1073741914_1090 (size=12301) 2024-11-16T09:58:16,787 INFO [RS_FLUSH_OPERATIONS-regionserver/037c58a19ddb:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=376 (bloomFilter=true), to=hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/d9e28756f78b0555a6733ffbf6ab98bd/.tmp/C/98262bb2b6fb41ecbf6e44ee204fa711 2024-11-16T09:58:16,794 DEBUG [RS_FLUSH_OPERATIONS-regionserver/037c58a19ddb:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/d9e28756f78b0555a6733ffbf6ab98bd/.tmp/A/193f043412fd40d3989d5e43a5329301 as hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/d9e28756f78b0555a6733ffbf6ab98bd/A/193f043412fd40d3989d5e43a5329301 2024-11-16T09:58:16,800 INFO [RS_FLUSH_OPERATIONS-regionserver/037c58a19ddb:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/d9e28756f78b0555a6733ffbf6ab98bd/A/193f043412fd40d3989d5e43a5329301, entries=150, sequenceid=376, filesize=12.0 K 2024-11-16T09:58:16,801 DEBUG [RS_FLUSH_OPERATIONS-regionserver/037c58a19ddb:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/d9e28756f78b0555a6733ffbf6ab98bd/.tmp/B/948594f9b3c641f28c66b71ad0b18386 as hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/d9e28756f78b0555a6733ffbf6ab98bd/B/948594f9b3c641f28c66b71ad0b18386 2024-11-16T09:58:16,806 INFO [RS_FLUSH_OPERATIONS-regionserver/037c58a19ddb:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/d9e28756f78b0555a6733ffbf6ab98bd/B/948594f9b3c641f28c66b71ad0b18386, entries=150, sequenceid=376, filesize=12.0 K 2024-11-16T09:58:16,807 DEBUG [RS_FLUSH_OPERATIONS-regionserver/037c58a19ddb:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/d9e28756f78b0555a6733ffbf6ab98bd/.tmp/C/98262bb2b6fb41ecbf6e44ee204fa711 as hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/d9e28756f78b0555a6733ffbf6ab98bd/C/98262bb2b6fb41ecbf6e44ee204fa711 2024-11-16T09:58:16,812 INFO [RS_FLUSH_OPERATIONS-regionserver/037c58a19ddb:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/d9e28756f78b0555a6733ffbf6ab98bd/C/98262bb2b6fb41ecbf6e44ee204fa711, entries=150, sequenceid=376, filesize=12.0 K 2024-11-16T09:58:16,813 INFO [RS_FLUSH_OPERATIONS-regionserver/037c58a19ddb:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegion(3140): Finished flush of dataSize ~134.18 KB/137400, heapSize ~352.27 KB/360720, currentSize=0 B/0 for d9e28756f78b0555a6733ffbf6ab98bd in 92ms, sequenceid=376, compaction requested=true 2024-11-16T09:58:16,813 DEBUG [RS_FLUSH_OPERATIONS-regionserver/037c58a19ddb:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegion(2603): Flush status journal for d9e28756f78b0555a6733ffbf6ab98bd: 2024-11-16T09:58:16,813 DEBUG [RS_FLUSH_OPERATIONS-regionserver/037c58a19ddb:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731751082172.d9e28756f78b0555a6733ffbf6ab98bd. 2024-11-16T09:58:16,813 DEBUG [RS_FLUSH_OPERATIONS-regionserver/037c58a19ddb:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=16 2024-11-16T09:58:16,814 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37877 {}] master.HMaster(4169): Remote procedure done, pid=16 2024-11-16T09:58:16,819 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=16, resume processing ppid=15 2024-11-16T09:58:16,819 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=16, ppid=15, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 718 msec 2024-11-16T09:58:16,821 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=15, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=15, table=TestAcidGuarantees in 725 msec 2024-11-16T09:58:16,988 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-16T09:58:16,988 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] regionserver.HRegion(8855): Flush requested on d9e28756f78b0555a6733ffbf6ab98bd 2024-11-16T09:58:16,988 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing d9e28756f78b0555a6733ffbf6ab98bd 3/3 column families, dataSize=60.38 KB heapSize=158.95 KB 2024-11-16T09:58:16,995 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/d9e28756f78b0555a6733ffbf6ab98bd/.tmp/A/98ef79933cf949c4aaba70bd1f16827e is 50, key is test_row_0/A:col10/1731751096987/Put/seqid=0 2024-11-16T09:58:17,010 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37591 is added to blk_1073741915_1091 (size=12301) 2024-11-16T09:58:17,011 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=388 (bloomFilter=true), to=hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/d9e28756f78b0555a6733ffbf6ab98bd/.tmp/A/98ef79933cf949c4aaba70bd1f16827e 2024-11-16T09:58:17,024 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/d9e28756f78b0555a6733ffbf6ab98bd/.tmp/B/3ef734138e094b948fd60c7570996e0d is 50, key is test_row_0/B:col10/1731751096987/Put/seqid=0 2024-11-16T09:58:17,029 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9e28756f78b0555a6733ffbf6ab98bd, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T09:58:17,030 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] ipc.CallRunner(138): callId: 108 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39638 deadline: 1731751157022, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9e28756f78b0555a6733ffbf6ab98bd, server=037c58a19ddb,37267,1731751077844 2024-11-16T09:58:17,030 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9e28756f78b0555a6733ffbf6ab98bd, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T09:58:17,030 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] ipc.CallRunner(138): callId: 118 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39620 deadline: 1731751157027, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9e28756f78b0555a6733ffbf6ab98bd, server=037c58a19ddb,37267,1731751077844 2024-11-16T09:58:17,030 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9e28756f78b0555a6733ffbf6ab98bd, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T09:58:17,031 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] ipc.CallRunner(138): callId: 108 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39630 deadline: 1731751157028, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9e28756f78b0555a6733ffbf6ab98bd, server=037c58a19ddb,37267,1731751077844 2024-11-16T09:58:17,031 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731751082172.d9e28756f78b0555a6733ffbf6ab98bd., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 , the old value is region=TestAcidGuarantees,,1731751082172.d9e28756f78b0555a6733ffbf6ab98bd., hostname=037c58a19ddb,37267,1731751077844, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9e28756f78b0555a6733ffbf6ab98bd, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:58:17,031 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731751082172.d9e28756f78b0555a6733ffbf6ab98bd., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9e28756f78b0555a6733ffbf6ab98bd, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:58:17,031 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731751082172.d9e28756f78b0555a6733ffbf6ab98bd., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 because the exception is null or not the one we care about 2024-11-16T09:58:17,031 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731751082172.d9e28756f78b0555a6733ffbf6ab98bd., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 , the old value is region=TestAcidGuarantees,,1731751082172.d9e28756f78b0555a6733ffbf6ab98bd., hostname=037c58a19ddb,37267,1731751077844, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9e28756f78b0555a6733ffbf6ab98bd, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:58:17,031 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731751082172.d9e28756f78b0555a6733ffbf6ab98bd., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9e28756f78b0555a6733ffbf6ab98bd, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:58:17,031 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731751082172.d9e28756f78b0555a6733ffbf6ab98bd., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 because the exception is null or not the one we care about 2024-11-16T09:58:17,032 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731751082172.d9e28756f78b0555a6733ffbf6ab98bd., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 , the old value is region=TestAcidGuarantees,,1731751082172.d9e28756f78b0555a6733ffbf6ab98bd., hostname=037c58a19ddb,37267,1731751077844, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9e28756f78b0555a6733ffbf6ab98bd, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:58:17,032 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731751082172.d9e28756f78b0555a6733ffbf6ab98bd., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9e28756f78b0555a6733ffbf6ab98bd, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:58:17,032 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731751082172.d9e28756f78b0555a6733ffbf6ab98bd., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 because the exception is null or not the one we care about 2024-11-16T09:58:17,034 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9e28756f78b0555a6733ffbf6ab98bd, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T09:58:17,035 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] ipc.CallRunner(138): callId: 118 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39654 deadline: 1731751157032, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9e28756f78b0555a6733ffbf6ab98bd, server=037c58a19ddb,37267,1731751077844 2024-11-16T09:58:17,035 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731751082172.d9e28756f78b0555a6733ffbf6ab98bd., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 , the old value is region=TestAcidGuarantees,,1731751082172.d9e28756f78b0555a6733ffbf6ab98bd., hostname=037c58a19ddb,37267,1731751077844, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9e28756f78b0555a6733ffbf6ab98bd, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:58:17,035 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731751082172.d9e28756f78b0555a6733ffbf6ab98bd., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9e28756f78b0555a6733ffbf6ab98bd, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:58:17,035 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9e28756f78b0555a6733ffbf6ab98bd, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T09:58:17,035 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731751082172.d9e28756f78b0555a6733ffbf6ab98bd., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 because the exception is null or not the one we care about 2024-11-16T09:58:17,035 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] ipc.CallRunner(138): callId: 116 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39696 deadline: 1731751157033, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9e28756f78b0555a6733ffbf6ab98bd, server=037c58a19ddb,37267,1731751077844 2024-11-16T09:58:17,035 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731751082172.d9e28756f78b0555a6733ffbf6ab98bd., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 , the old value is region=TestAcidGuarantees,,1731751082172.d9e28756f78b0555a6733ffbf6ab98bd., hostname=037c58a19ddb,37267,1731751077844, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9e28756f78b0555a6733ffbf6ab98bd, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:58:17,035 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731751082172.d9e28756f78b0555a6733ffbf6ab98bd., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9e28756f78b0555a6733ffbf6ab98bd, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:58:17,036 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731751082172.d9e28756f78b0555a6733ffbf6ab98bd., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 because the exception is null or not the one we care about 2024-11-16T09:58:17,043 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37591 is added to blk_1073741916_1092 (size=12301) 2024-11-16T09:58:17,044 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=388 (bloomFilter=true), to=hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/d9e28756f78b0555a6733ffbf6ab98bd/.tmp/B/3ef734138e094b948fd60c7570996e0d 2024-11-16T09:58:17,056 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/d9e28756f78b0555a6733ffbf6ab98bd/.tmp/C/82d83ddf384a4de2b8ba2f34f81b7bca is 50, key is test_row_0/C:col10/1731751096987/Put/seqid=0 2024-11-16T09:58:17,061 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37591 is added to blk_1073741917_1093 (size=12301) 2024-11-16T09:58:17,062 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=388 (bloomFilter=true), to=hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/d9e28756f78b0555a6733ffbf6ab98bd/.tmp/C/82d83ddf384a4de2b8ba2f34f81b7bca 2024-11-16T09:58:17,071 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/d9e28756f78b0555a6733ffbf6ab98bd/.tmp/A/98ef79933cf949c4aaba70bd1f16827e as hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/d9e28756f78b0555a6733ffbf6ab98bd/A/98ef79933cf949c4aaba70bd1f16827e 2024-11-16T09:58:17,077 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/d9e28756f78b0555a6733ffbf6ab98bd/A/98ef79933cf949c4aaba70bd1f16827e, entries=150, sequenceid=388, filesize=12.0 K 2024-11-16T09:58:17,078 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/d9e28756f78b0555a6733ffbf6ab98bd/.tmp/B/3ef734138e094b948fd60c7570996e0d as hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/d9e28756f78b0555a6733ffbf6ab98bd/B/3ef734138e094b948fd60c7570996e0d 2024-11-16T09:58:17,086 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/d9e28756f78b0555a6733ffbf6ab98bd/B/3ef734138e094b948fd60c7570996e0d, entries=150, sequenceid=388, filesize=12.0 K 2024-11-16T09:58:17,087 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/d9e28756f78b0555a6733ffbf6ab98bd/.tmp/C/82d83ddf384a4de2b8ba2f34f81b7bca as hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/d9e28756f78b0555a6733ffbf6ab98bd/C/82d83ddf384a4de2b8ba2f34f81b7bca 2024-11-16T09:58:17,094 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/d9e28756f78b0555a6733ffbf6ab98bd/C/82d83ddf384a4de2b8ba2f34f81b7bca, entries=150, sequenceid=388, filesize=12.0 K 2024-11-16T09:58:17,095 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~158.91 KB/162720, currentSize=147.60 KB/151140 for d9e28756f78b0555a6733ffbf6ab98bd in 107ms, sequenceid=388, compaction requested=true 2024-11-16T09:58:17,096 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for d9e28756f78b0555a6733ffbf6ab98bd: 2024-11-16T09:58:17,096 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store d9e28756f78b0555a6733ffbf6ab98bd:A, priority=-2147483648, current under compaction store size is 1 2024-11-16T09:58:17,096 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-16T09:58:17,096 DEBUG [RS:0;037c58a19ddb:37267-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-16T09:58:17,096 DEBUG [RS:0;037c58a19ddb:37267-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-16T09:58:17,096 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store d9e28756f78b0555a6733ffbf6ab98bd:B, priority=-2147483648, current under compaction store size is 2 2024-11-16T09:58:17,096 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-16T09:58:17,096 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store d9e28756f78b0555a6733ffbf6ab98bd:C, priority=-2147483648, current under compaction store size is 3 2024-11-16T09:58:17,096 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-16T09:58:17,098 DEBUG [RS:0;037c58a19ddb:37267-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 49988 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-16T09:58:17,098 DEBUG [RS:0;037c58a19ddb:37267-shortCompactions-0 {}] regionserver.HStore(1541): d9e28756f78b0555a6733ffbf6ab98bd/A is initiating minor compaction (all files) 2024-11-16T09:58:17,098 INFO [RS:0;037c58a19ddb:37267-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of d9e28756f78b0555a6733ffbf6ab98bd/A in TestAcidGuarantees,,1731751082172.d9e28756f78b0555a6733ffbf6ab98bd. 2024-11-16T09:58:17,098 INFO [RS:0;037c58a19ddb:37267-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/d9e28756f78b0555a6733ffbf6ab98bd/A/32643ed1b9394c95a86ca96dcf7364dd, hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/d9e28756f78b0555a6733ffbf6ab98bd/A/d1f88cc47aef44b2a64aed41364c5430, hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/d9e28756f78b0555a6733ffbf6ab98bd/A/193f043412fd40d3989d5e43a5329301, hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/d9e28756f78b0555a6733ffbf6ab98bd/A/98ef79933cf949c4aaba70bd1f16827e] into tmpdir=hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/d9e28756f78b0555a6733ffbf6ab98bd/.tmp, totalSize=48.8 K 2024-11-16T09:58:17,099 DEBUG [RS:0;037c58a19ddb:37267-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 49988 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-16T09:58:17,099 DEBUG [RS:0;037c58a19ddb:37267-longCompactions-0 {}] regionserver.HStore(1541): d9e28756f78b0555a6733ffbf6ab98bd/B is initiating minor compaction (all files) 2024-11-16T09:58:17,099 INFO [RS:0;037c58a19ddb:37267-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of d9e28756f78b0555a6733ffbf6ab98bd/B in TestAcidGuarantees,,1731751082172.d9e28756f78b0555a6733ffbf6ab98bd. 2024-11-16T09:58:17,099 INFO [RS:0;037c58a19ddb:37267-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/d9e28756f78b0555a6733ffbf6ab98bd/B/b468b9f5c55b4a87ad1e6ee8500a6260, hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/d9e28756f78b0555a6733ffbf6ab98bd/B/3fade5830bae4cc28018c7702bc3b811, hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/d9e28756f78b0555a6733ffbf6ab98bd/B/948594f9b3c641f28c66b71ad0b18386, hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/d9e28756f78b0555a6733ffbf6ab98bd/B/3ef734138e094b948fd60c7570996e0d] into tmpdir=hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/d9e28756f78b0555a6733ffbf6ab98bd/.tmp, totalSize=48.8 K 2024-11-16T09:58:17,099 DEBUG [RS:0;037c58a19ddb:37267-shortCompactions-0 {}] compactions.Compactor(225): Compacting 32643ed1b9394c95a86ca96dcf7364dd, keycount=150, bloomtype=ROW, size=12.8 K, encoding=NONE, compression=NONE, seqNum=336, earliestPutTs=1731751093973 2024-11-16T09:58:17,100 DEBUG [RS:0;037c58a19ddb:37267-longCompactions-0 {}] compactions.Compactor(225): Compacting b468b9f5c55b4a87ad1e6ee8500a6260, keycount=150, bloomtype=ROW, size=12.8 K, encoding=NONE, compression=NONE, seqNum=336, earliestPutTs=1731751093973 2024-11-16T09:58:17,100 DEBUG [RS:0;037c58a19ddb:37267-shortCompactions-0 {}] compactions.Compactor(225): Compacting d1f88cc47aef44b2a64aed41364c5430, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=353, earliestPutTs=1731751094632 2024-11-16T09:58:17,101 DEBUG [RS:0;037c58a19ddb:37267-longCompactions-0 {}] compactions.Compactor(225): Compacting 3fade5830bae4cc28018c7702bc3b811, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=353, earliestPutTs=1731751094632 2024-11-16T09:58:17,101 DEBUG [RS:0;037c58a19ddb:37267-shortCompactions-0 {}] compactions.Compactor(225): Compacting 193f043412fd40d3989d5e43a5329301, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=376, earliestPutTs=1731751095827 2024-11-16T09:58:17,102 DEBUG [RS:0;037c58a19ddb:37267-longCompactions-0 {}] compactions.Compactor(225): Compacting 948594f9b3c641f28c66b71ad0b18386, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=376, earliestPutTs=1731751095827 2024-11-16T09:58:17,102 DEBUG [RS:0;037c58a19ddb:37267-shortCompactions-0 {}] compactions.Compactor(225): Compacting 98ef79933cf949c4aaba70bd1f16827e, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=388, earliestPutTs=1731751096983 2024-11-16T09:58:17,103 DEBUG [RS:0;037c58a19ddb:37267-longCompactions-0 {}] compactions.Compactor(225): Compacting 3ef734138e094b948fd60c7570996e0d, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=388, earliestPutTs=1731751096983 2024-11-16T09:58:17,117 INFO [RS:0;037c58a19ddb:37267-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): d9e28756f78b0555a6733ffbf6ab98bd#A#compaction#81 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-16T09:58:17,118 DEBUG [RS:0;037c58a19ddb:37267-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/d9e28756f78b0555a6733ffbf6ab98bd/.tmp/A/2eb7658b24074f5f91f101c2bef2e6c7 is 50, key is test_row_0/A:col10/1731751096987/Put/seqid=0 2024-11-16T09:58:17,122 INFO [RS:0;037c58a19ddb:37267-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): d9e28756f78b0555a6733ffbf6ab98bd#B#compaction#82 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-16T09:58:17,123 DEBUG [RS:0;037c58a19ddb:37267-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/d9e28756f78b0555a6733ffbf6ab98bd/.tmp/B/a1d07b488d594516be54d1ab5a5ccf78 is 50, key is test_row_0/B:col10/1731751096987/Put/seqid=0 2024-11-16T09:58:17,134 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37591 is added to blk_1073741918_1094 (size=13221) 2024-11-16T09:58:17,135 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] regionserver.HRegion(8855): Flush requested on d9e28756f78b0555a6733ffbf6ab98bd 2024-11-16T09:58:17,135 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-16T09:58:17,135 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37591 is added to blk_1073741919_1095 (size=13221) 2024-11-16T09:58:17,135 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing d9e28756f78b0555a6733ffbf6ab98bd 3/3 column families, dataSize=154.31 KB heapSize=405.05 KB 2024-11-16T09:58:17,142 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/d9e28756f78b0555a6733ffbf6ab98bd/.tmp/A/4bbd180dd1ad4a2884e953f2b35b43be is 50, key is test_row_0/A:col10/1731751097133/Put/seqid=0 2024-11-16T09:58:17,147 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37591 is added to blk_1073741920_1096 (size=17181) 2024-11-16T09:58:17,147 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=53.67 KB at sequenceid=415 (bloomFilter=true), to=hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/d9e28756f78b0555a6733ffbf6ab98bd/.tmp/A/4bbd180dd1ad4a2884e953f2b35b43be 2024-11-16T09:58:17,150 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9e28756f78b0555a6733ffbf6ab98bd, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T09:58:17,151 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] ipc.CallRunner(138): callId: 112 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39630 deadline: 1731751157147, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9e28756f78b0555a6733ffbf6ab98bd, server=037c58a19ddb,37267,1731751077844 2024-11-16T09:58:17,151 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731751082172.d9e28756f78b0555a6733ffbf6ab98bd., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 , the old value is region=TestAcidGuarantees,,1731751082172.d9e28756f78b0555a6733ffbf6ab98bd., hostname=037c58a19ddb,37267,1731751077844, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9e28756f78b0555a6733ffbf6ab98bd, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:58:17,151 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731751082172.d9e28756f78b0555a6733ffbf6ab98bd., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9e28756f78b0555a6733ffbf6ab98bd, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:58:17,151 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9e28756f78b0555a6733ffbf6ab98bd, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T09:58:17,151 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731751082172.d9e28756f78b0555a6733ffbf6ab98bd., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 because the exception is null or not the one we care about 2024-11-16T09:58:17,151 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] ipc.CallRunner(138): callId: 112 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39638 deadline: 1731751157147, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9e28756f78b0555a6733ffbf6ab98bd, server=037c58a19ddb,37267,1731751077844 2024-11-16T09:58:17,152 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731751082172.d9e28756f78b0555a6733ffbf6ab98bd., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 , the old value is region=TestAcidGuarantees,,1731751082172.d9e28756f78b0555a6733ffbf6ab98bd., hostname=037c58a19ddb,37267,1731751077844, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9e28756f78b0555a6733ffbf6ab98bd, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:58:17,152 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9e28756f78b0555a6733ffbf6ab98bd, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T09:58:17,152 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731751082172.d9e28756f78b0555a6733ffbf6ab98bd., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9e28756f78b0555a6733ffbf6ab98bd, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:58:17,152 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731751082172.d9e28756f78b0555a6733ffbf6ab98bd., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 because the exception is null or not the one we care about 2024-11-16T09:58:17,152 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] ipc.CallRunner(138): callId: 120 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39654 deadline: 1731751157148, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9e28756f78b0555a6733ffbf6ab98bd, server=037c58a19ddb,37267,1731751077844 2024-11-16T09:58:17,152 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731751082172.d9e28756f78b0555a6733ffbf6ab98bd., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 , the old value is region=TestAcidGuarantees,,1731751082172.d9e28756f78b0555a6733ffbf6ab98bd., hostname=037c58a19ddb,37267,1731751077844, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9e28756f78b0555a6733ffbf6ab98bd, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:58:17,152 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9e28756f78b0555a6733ffbf6ab98bd, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T09:58:17,152 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731751082172.d9e28756f78b0555a6733ffbf6ab98bd., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9e28756f78b0555a6733ffbf6ab98bd, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:58:17,152 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731751082172.d9e28756f78b0555a6733ffbf6ab98bd., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 because the exception is null or not the one we care about 2024-11-16T09:58:17,152 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] ipc.CallRunner(138): callId: 120 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39620 deadline: 1731751157148, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9e28756f78b0555a6733ffbf6ab98bd, server=037c58a19ddb,37267,1731751077844 2024-11-16T09:58:17,153 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731751082172.d9e28756f78b0555a6733ffbf6ab98bd., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 , the old value is region=TestAcidGuarantees,,1731751082172.d9e28756f78b0555a6733ffbf6ab98bd., hostname=037c58a19ddb,37267,1731751077844, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9e28756f78b0555a6733ffbf6ab98bd, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:58:17,153 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731751082172.d9e28756f78b0555a6733ffbf6ab98bd., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9e28756f78b0555a6733ffbf6ab98bd, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:58:17,153 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9e28756f78b0555a6733ffbf6ab98bd, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T09:58:17,153 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731751082172.d9e28756f78b0555a6733ffbf6ab98bd., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 because the exception is null or not the one we care about 2024-11-16T09:58:17,153 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] ipc.CallRunner(138): callId: 118 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39696 deadline: 1731751157149, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9e28756f78b0555a6733ffbf6ab98bd, server=037c58a19ddb,37267,1731751077844 2024-11-16T09:58:17,153 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731751082172.d9e28756f78b0555a6733ffbf6ab98bd., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 , the old value is region=TestAcidGuarantees,,1731751082172.d9e28756f78b0555a6733ffbf6ab98bd., hostname=037c58a19ddb,37267,1731751077844, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9e28756f78b0555a6733ffbf6ab98bd, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:58:17,153 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731751082172.d9e28756f78b0555a6733ffbf6ab98bd., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9e28756f78b0555a6733ffbf6ab98bd, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:58:17,153 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731751082172.d9e28756f78b0555a6733ffbf6ab98bd., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 because the exception is null or not the one we care about 2024-11-16T09:58:17,159 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/d9e28756f78b0555a6733ffbf6ab98bd/.tmp/B/b63e28d4eb564c249b118b3622ecad37 is 50, key is test_row_0/B:col10/1731751097133/Put/seqid=0 2024-11-16T09:58:17,174 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37591 is added to blk_1073741921_1097 (size=12301) 2024-11-16T09:58:17,175 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=53.67 KB at sequenceid=415 (bloomFilter=true), to=hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/d9e28756f78b0555a6733ffbf6ab98bd/.tmp/B/b63e28d4eb564c249b118b3622ecad37 2024-11-16T09:58:17,199 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/d9e28756f78b0555a6733ffbf6ab98bd/.tmp/C/6abf151a60c5402f91aa7af92dc02406 is 50, key is test_row_0/C:col10/1731751097133/Put/seqid=0 2024-11-16T09:58:17,211 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37591 is added to blk_1073741922_1098 (size=12301) 2024-11-16T09:58:17,232 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37877 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=15 2024-11-16T09:58:17,233 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-16T09:58:17,233 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37877 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-16T09:58:17,234 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37877 {}] procedure2.ProcedureExecutor(1139): Stored pid=17, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=17, table=TestAcidGuarantees 2024-11-16T09:58:17,235 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=17, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=17, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-16T09:58:17,235 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37877 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=17 2024-11-16T09:58:17,236 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=17, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=17, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-16T09:58:17,236 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=18, ppid=17, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-16T09:58:17,255 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9e28756f78b0555a6733ffbf6ab98bd, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T09:58:17,255 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] ipc.CallRunner(138): callId: 113 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39630 deadline: 1731751157253, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9e28756f78b0555a6733ffbf6ab98bd, server=037c58a19ddb,37267,1731751077844 2024-11-16T09:58:17,256 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731751082172.d9e28756f78b0555a6733ffbf6ab98bd., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 , the old value is region=TestAcidGuarantees,,1731751082172.d9e28756f78b0555a6733ffbf6ab98bd., hostname=037c58a19ddb,37267,1731751077844, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9e28756f78b0555a6733ffbf6ab98bd, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:58:17,256 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731751082172.d9e28756f78b0555a6733ffbf6ab98bd., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9e28756f78b0555a6733ffbf6ab98bd, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:58:17,256 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731751082172.d9e28756f78b0555a6733ffbf6ab98bd., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 because the exception is null or not the one we care about 2024-11-16T09:58:17,270 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9e28756f78b0555a6733ffbf6ab98bd, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T09:58:17,270 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] ipc.CallRunner(138): callId: 113 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39638 deadline: 1731751157263, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9e28756f78b0555a6733ffbf6ab98bd, server=037c58a19ddb,37267,1731751077844 2024-11-16T09:58:17,270 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9e28756f78b0555a6733ffbf6ab98bd, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T09:58:17,271 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] ipc.CallRunner(138): callId: 121 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39654 deadline: 1731751157263, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9e28756f78b0555a6733ffbf6ab98bd, server=037c58a19ddb,37267,1731751077844 2024-11-16T09:58:17,271 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731751082172.d9e28756f78b0555a6733ffbf6ab98bd., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 , the old value is region=TestAcidGuarantees,,1731751082172.d9e28756f78b0555a6733ffbf6ab98bd., hostname=037c58a19ddb,37267,1731751077844, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9e28756f78b0555a6733ffbf6ab98bd, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:58:17,271 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731751082172.d9e28756f78b0555a6733ffbf6ab98bd., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9e28756f78b0555a6733ffbf6ab98bd, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:58:17,271 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9e28756f78b0555a6733ffbf6ab98bd, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T09:58:17,271 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9e28756f78b0555a6733ffbf6ab98bd, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T09:58:17,271 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731751082172.d9e28756f78b0555a6733ffbf6ab98bd., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 because the exception is null or not the one we care about 2024-11-16T09:58:17,271 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] ipc.CallRunner(138): callId: 119 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39696 deadline: 1731751157263, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9e28756f78b0555a6733ffbf6ab98bd, server=037c58a19ddb,37267,1731751077844 2024-11-16T09:58:17,271 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] ipc.CallRunner(138): callId: 121 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39620 deadline: 1731751157263, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9e28756f78b0555a6733ffbf6ab98bd, server=037c58a19ddb,37267,1731751077844 2024-11-16T09:58:17,271 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731751082172.d9e28756f78b0555a6733ffbf6ab98bd., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 , the old value is region=TestAcidGuarantees,,1731751082172.d9e28756f78b0555a6733ffbf6ab98bd., hostname=037c58a19ddb,37267,1731751077844, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9e28756f78b0555a6733ffbf6ab98bd, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:58:17,271 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731751082172.d9e28756f78b0555a6733ffbf6ab98bd., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 , the old value is region=TestAcidGuarantees,,1731751082172.d9e28756f78b0555a6733ffbf6ab98bd., hostname=037c58a19ddb,37267,1731751077844, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9e28756f78b0555a6733ffbf6ab98bd, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:58:17,271 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731751082172.d9e28756f78b0555a6733ffbf6ab98bd., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 , the old value is region=TestAcidGuarantees,,1731751082172.d9e28756f78b0555a6733ffbf6ab98bd., hostname=037c58a19ddb,37267,1731751077844, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9e28756f78b0555a6733ffbf6ab98bd, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:58:17,272 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731751082172.d9e28756f78b0555a6733ffbf6ab98bd., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9e28756f78b0555a6733ffbf6ab98bd, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:58:17,272 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731751082172.d9e28756f78b0555a6733ffbf6ab98bd., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9e28756f78b0555a6733ffbf6ab98bd, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:58:17,272 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731751082172.d9e28756f78b0555a6733ffbf6ab98bd., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9e28756f78b0555a6733ffbf6ab98bd, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:58:17,272 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731751082172.d9e28756f78b0555a6733ffbf6ab98bd., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 because the exception is null or not the one we care about 2024-11-16T09:58:17,272 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731751082172.d9e28756f78b0555a6733ffbf6ab98bd., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 because the exception is null or not the one we care about 2024-11-16T09:58:17,272 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731751082172.d9e28756f78b0555a6733ffbf6ab98bd., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 because the exception is null or not the one we care about 2024-11-16T09:58:17,343 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37877 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=17 2024-11-16T09:58:17,389 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=37267 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=18 2024-11-16T09:58:17,390 DEBUG [RS_FLUSH_OPERATIONS-regionserver/037c58a19ddb:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731751082172.d9e28756f78b0555a6733ffbf6ab98bd. 2024-11-16T09:58:17,390 DEBUG [RS_FLUSH_OPERATIONS-regionserver/037c58a19ddb:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731751082172.d9e28756f78b0555a6733ffbf6ab98bd. as already flushing 2024-11-16T09:58:17,390 DEBUG [RS_FLUSH_OPERATIONS-regionserver/037c58a19ddb:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731751082172.d9e28756f78b0555a6733ffbf6ab98bd. 2024-11-16T09:58:17,390 ERROR [RS_FLUSH_OPERATIONS-regionserver/037c58a19ddb:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] handler.RSProcedureHandler(58): pid=18 java.io.IOException: Unable to complete flush {ENCODED => d9e28756f78b0555a6733ffbf6ab98bd, NAME => 'TestAcidGuarantees,,1731751082172.d9e28756f78b0555a6733ffbf6ab98bd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T09:58:17,390 DEBUG [RS_FLUSH_OPERATIONS-regionserver/037c58a19ddb:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=18 java.io.IOException: Unable to complete flush {ENCODED => d9e28756f78b0555a6733ffbf6ab98bd, NAME => 'TestAcidGuarantees,,1731751082172.d9e28756f78b0555a6733ffbf6ab98bd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T09:58:17,391 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37877 {}] master.HMaster(4177): Remote procedure failed, pid=18 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d9e28756f78b0555a6733ffbf6ab98bd, NAME => 'TestAcidGuarantees,,1731751082172.d9e28756f78b0555a6733ffbf6ab98bd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d9e28756f78b0555a6733ffbf6ab98bd, NAME => 'TestAcidGuarantees,,1731751082172.d9e28756f78b0555a6733ffbf6ab98bd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T09:58:17,468 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9e28756f78b0555a6733ffbf6ab98bd, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T09:58:17,468 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] ipc.CallRunner(138): callId: 114 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39630 deadline: 1731751157463, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9e28756f78b0555a6733ffbf6ab98bd, server=037c58a19ddb,37267,1731751077844 2024-11-16T09:58:17,469 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731751082172.d9e28756f78b0555a6733ffbf6ab98bd., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 , the old value is region=TestAcidGuarantees,,1731751082172.d9e28756f78b0555a6733ffbf6ab98bd., hostname=037c58a19ddb,37267,1731751077844, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9e28756f78b0555a6733ffbf6ab98bd, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:58:17,469 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731751082172.d9e28756f78b0555a6733ffbf6ab98bd., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9e28756f78b0555a6733ffbf6ab98bd, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:58:17,469 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731751082172.d9e28756f78b0555a6733ffbf6ab98bd., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 because the exception is null or not the one we care about 2024-11-16T09:58:17,484 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9e28756f78b0555a6733ffbf6ab98bd, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T09:58:17,484 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] ipc.CallRunner(138): callId: 122 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39620 deadline: 1731751157483, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9e28756f78b0555a6733ffbf6ab98bd, server=037c58a19ddb,37267,1731751077844 2024-11-16T09:58:17,484 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731751082172.d9e28756f78b0555a6733ffbf6ab98bd., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 , the old value is region=TestAcidGuarantees,,1731751082172.d9e28756f78b0555a6733ffbf6ab98bd., hostname=037c58a19ddb,37267,1731751077844, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9e28756f78b0555a6733ffbf6ab98bd, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:58:17,484 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731751082172.d9e28756f78b0555a6733ffbf6ab98bd., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9e28756f78b0555a6733ffbf6ab98bd, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:58:17,484 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9e28756f78b0555a6733ffbf6ab98bd, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T09:58:17,485 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731751082172.d9e28756f78b0555a6733ffbf6ab98bd., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 because the exception is null or not the one we care about 2024-11-16T09:58:17,485 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] ipc.CallRunner(138): callId: 114 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39638 deadline: 1731751157483, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9e28756f78b0555a6733ffbf6ab98bd, server=037c58a19ddb,37267,1731751077844 2024-11-16T09:58:17,485 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731751082172.d9e28756f78b0555a6733ffbf6ab98bd., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 , the old value is region=TestAcidGuarantees,,1731751082172.d9e28756f78b0555a6733ffbf6ab98bd., hostname=037c58a19ddb,37267,1731751077844, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9e28756f78b0555a6733ffbf6ab98bd, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:58:17,485 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731751082172.d9e28756f78b0555a6733ffbf6ab98bd., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9e28756f78b0555a6733ffbf6ab98bd, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:58:17,485 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731751082172.d9e28756f78b0555a6733ffbf6ab98bd., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 because the exception is null or not the one we care about 2024-11-16T09:58:17,485 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9e28756f78b0555a6733ffbf6ab98bd, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T09:58:17,485 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] ipc.CallRunner(138): callId: 120 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39696 deadline: 1731751157483, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9e28756f78b0555a6733ffbf6ab98bd, server=037c58a19ddb,37267,1731751077844 2024-11-16T09:58:17,485 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9e28756f78b0555a6733ffbf6ab98bd, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T09:58:17,485 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] ipc.CallRunner(138): callId: 122 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39654 deadline: 1731751157483, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9e28756f78b0555a6733ffbf6ab98bd, server=037c58a19ddb,37267,1731751077844 2024-11-16T09:58:17,486 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731751082172.d9e28756f78b0555a6733ffbf6ab98bd., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 , the old value is region=TestAcidGuarantees,,1731751082172.d9e28756f78b0555a6733ffbf6ab98bd., hostname=037c58a19ddb,37267,1731751077844, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9e28756f78b0555a6733ffbf6ab98bd, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:58:17,486 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731751082172.d9e28756f78b0555a6733ffbf6ab98bd., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9e28756f78b0555a6733ffbf6ab98bd, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:58:17,486 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731751082172.d9e28756f78b0555a6733ffbf6ab98bd., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 because the exception is null or not the one we care about 2024-11-16T09:58:17,488 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731751082172.d9e28756f78b0555a6733ffbf6ab98bd., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 , the old value is region=TestAcidGuarantees,,1731751082172.d9e28756f78b0555a6733ffbf6ab98bd., hostname=037c58a19ddb,37267,1731751077844, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9e28756f78b0555a6733ffbf6ab98bd, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:58:17,488 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731751082172.d9e28756f78b0555a6733ffbf6ab98bd., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9e28756f78b0555a6733ffbf6ab98bd, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:58:17,488 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731751082172.d9e28756f78b0555a6733ffbf6ab98bd., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 because the exception is null or not the one we care about 2024-11-16T09:58:17,542 DEBUG [RS:0;037c58a19ddb:37267-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/d9e28756f78b0555a6733ffbf6ab98bd/.tmp/B/a1d07b488d594516be54d1ab5a5ccf78 as hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/d9e28756f78b0555a6733ffbf6ab98bd/B/a1d07b488d594516be54d1ab5a5ccf78 2024-11-16T09:58:17,543 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=37267 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=18 2024-11-16T09:58:17,543 DEBUG [RS:0;037c58a19ddb:37267-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/d9e28756f78b0555a6733ffbf6ab98bd/.tmp/A/2eb7658b24074f5f91f101c2bef2e6c7 as hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/d9e28756f78b0555a6733ffbf6ab98bd/A/2eb7658b24074f5f91f101c2bef2e6c7 2024-11-16T09:58:17,544 DEBUG [RS_FLUSH_OPERATIONS-regionserver/037c58a19ddb:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731751082172.d9e28756f78b0555a6733ffbf6ab98bd. 2024-11-16T09:58:17,544 DEBUG [RS_FLUSH_OPERATIONS-regionserver/037c58a19ddb:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731751082172.d9e28756f78b0555a6733ffbf6ab98bd. as already flushing 2024-11-16T09:58:17,544 DEBUG [RS_FLUSH_OPERATIONS-regionserver/037c58a19ddb:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731751082172.d9e28756f78b0555a6733ffbf6ab98bd. 2024-11-16T09:58:17,544 ERROR [RS_FLUSH_OPERATIONS-regionserver/037c58a19ddb:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] handler.RSProcedureHandler(58): pid=18 java.io.IOException: Unable to complete flush {ENCODED => d9e28756f78b0555a6733ffbf6ab98bd, NAME => 'TestAcidGuarantees,,1731751082172.d9e28756f78b0555a6733ffbf6ab98bd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T09:58:17,544 DEBUG [RS_FLUSH_OPERATIONS-regionserver/037c58a19ddb:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=18 java.io.IOException: Unable to complete flush {ENCODED => d9e28756f78b0555a6733ffbf6ab98bd, NAME => 'TestAcidGuarantees,,1731751082172.d9e28756f78b0555a6733ffbf6ab98bd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T09:58:17,545 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37877 {}] master.HMaster(4177): Remote procedure failed, pid=18 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d9e28756f78b0555a6733ffbf6ab98bd, NAME => 'TestAcidGuarantees,,1731751082172.d9e28756f78b0555a6733ffbf6ab98bd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d9e28756f78b0555a6733ffbf6ab98bd, NAME => 'TestAcidGuarantees,,1731751082172.d9e28756f78b0555a6733ffbf6ab98bd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T09:58:17,553 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37877 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=17 2024-11-16T09:58:17,554 INFO [RS:0;037c58a19ddb:37267-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in d9e28756f78b0555a6733ffbf6ab98bd/B of d9e28756f78b0555a6733ffbf6ab98bd into a1d07b488d594516be54d1ab5a5ccf78(size=12.9 K), total size for store is 12.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-16T09:58:17,554 DEBUG [RS:0;037c58a19ddb:37267-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for d9e28756f78b0555a6733ffbf6ab98bd: 2024-11-16T09:58:17,554 INFO [RS:0;037c58a19ddb:37267-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731751082172.d9e28756f78b0555a6733ffbf6ab98bd., storeName=d9e28756f78b0555a6733ffbf6ab98bd/B, priority=12, startTime=1731751097096; duration=0sec 2024-11-16T09:58:17,554 DEBUG [RS:0;037c58a19ddb:37267-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-16T09:58:17,554 DEBUG [RS:0;037c58a19ddb:37267-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: d9e28756f78b0555a6733ffbf6ab98bd:B 2024-11-16T09:58:17,554 DEBUG [RS:0;037c58a19ddb:37267-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-16T09:58:17,555 INFO [RS:0;037c58a19ddb:37267-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in d9e28756f78b0555a6733ffbf6ab98bd/A of d9e28756f78b0555a6733ffbf6ab98bd into 2eb7658b24074f5f91f101c2bef2e6c7(size=12.9 K), total size for store is 12.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-16T09:58:17,555 DEBUG [RS:0;037c58a19ddb:37267-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for d9e28756f78b0555a6733ffbf6ab98bd: 2024-11-16T09:58:17,555 INFO [RS:0;037c58a19ddb:37267-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731751082172.d9e28756f78b0555a6733ffbf6ab98bd., storeName=d9e28756f78b0555a6733ffbf6ab98bd/A, priority=12, startTime=1731751097096; duration=0sec 2024-11-16T09:58:17,555 DEBUG [RS:0;037c58a19ddb:37267-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-16T09:58:17,555 DEBUG [RS:0;037c58a19ddb:37267-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: d9e28756f78b0555a6733ffbf6ab98bd:A 2024-11-16T09:58:17,557 DEBUG [RS:0;037c58a19ddb:37267-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 49988 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-16T09:58:17,557 DEBUG [RS:0;037c58a19ddb:37267-longCompactions-0 {}] regionserver.HStore(1541): d9e28756f78b0555a6733ffbf6ab98bd/C is initiating minor compaction (all files) 2024-11-16T09:58:17,557 INFO [RS:0;037c58a19ddb:37267-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of d9e28756f78b0555a6733ffbf6ab98bd/C in TestAcidGuarantees,,1731751082172.d9e28756f78b0555a6733ffbf6ab98bd. 2024-11-16T09:58:17,557 INFO [RS:0;037c58a19ddb:37267-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/d9e28756f78b0555a6733ffbf6ab98bd/C/167e57611134426aa91dd1a849a17aae, hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/d9e28756f78b0555a6733ffbf6ab98bd/C/0e51d1c5bd2e4038af191f3febcb368a, hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/d9e28756f78b0555a6733ffbf6ab98bd/C/98262bb2b6fb41ecbf6e44ee204fa711, hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/d9e28756f78b0555a6733ffbf6ab98bd/C/82d83ddf384a4de2b8ba2f34f81b7bca] into tmpdir=hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/d9e28756f78b0555a6733ffbf6ab98bd/.tmp, totalSize=48.8 K 2024-11-16T09:58:17,558 DEBUG [RS:0;037c58a19ddb:37267-longCompactions-0 {}] compactions.Compactor(225): Compacting 167e57611134426aa91dd1a849a17aae, keycount=150, bloomtype=ROW, size=12.8 K, encoding=NONE, compression=NONE, seqNum=336, earliestPutTs=1731751093973 2024-11-16T09:58:17,559 DEBUG [RS:0;037c58a19ddb:37267-longCompactions-0 {}] compactions.Compactor(225): Compacting 0e51d1c5bd2e4038af191f3febcb368a, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=353, earliestPutTs=1731751094632 2024-11-16T09:58:17,559 DEBUG [RS:0;037c58a19ddb:37267-longCompactions-0 {}] compactions.Compactor(225): Compacting 98262bb2b6fb41ecbf6e44ee204fa711, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=376, earliestPutTs=1731751095827 2024-11-16T09:58:17,559 DEBUG [RS:0;037c58a19ddb:37267-longCompactions-0 {}] compactions.Compactor(225): Compacting 82d83ddf384a4de2b8ba2f34f81b7bca, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=388, earliestPutTs=1731751096983 2024-11-16T09:58:17,571 INFO [RS:0;037c58a19ddb:37267-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): d9e28756f78b0555a6733ffbf6ab98bd#C#compaction#86 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-16T09:58:17,571 DEBUG [RS:0;037c58a19ddb:37267-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/d9e28756f78b0555a6733ffbf6ab98bd/.tmp/C/bafc286da4bb4b178dc9fe040d3351f4 is 50, key is test_row_0/C:col10/1731751096987/Put/seqid=0 2024-11-16T09:58:17,580 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37591 is added to blk_1073741923_1099 (size=13221) 2024-11-16T09:58:17,613 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=53.67 KB at sequenceid=415 (bloomFilter=true), to=hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/d9e28756f78b0555a6733ffbf6ab98bd/.tmp/C/6abf151a60c5402f91aa7af92dc02406 2024-11-16T09:58:17,621 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/d9e28756f78b0555a6733ffbf6ab98bd/.tmp/A/4bbd180dd1ad4a2884e953f2b35b43be as hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/d9e28756f78b0555a6733ffbf6ab98bd/A/4bbd180dd1ad4a2884e953f2b35b43be 2024-11-16T09:58:17,628 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/d9e28756f78b0555a6733ffbf6ab98bd/A/4bbd180dd1ad4a2884e953f2b35b43be, entries=250, sequenceid=415, filesize=16.8 K 2024-11-16T09:58:17,630 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/d9e28756f78b0555a6733ffbf6ab98bd/.tmp/B/b63e28d4eb564c249b118b3622ecad37 as hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/d9e28756f78b0555a6733ffbf6ab98bd/B/b63e28d4eb564c249b118b3622ecad37 2024-11-16T09:58:17,637 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/d9e28756f78b0555a6733ffbf6ab98bd/B/b63e28d4eb564c249b118b3622ecad37, entries=150, sequenceid=415, filesize=12.0 K 2024-11-16T09:58:17,639 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/d9e28756f78b0555a6733ffbf6ab98bd/.tmp/C/6abf151a60c5402f91aa7af92dc02406 as hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/d9e28756f78b0555a6733ffbf6ab98bd/C/6abf151a60c5402f91aa7af92dc02406 2024-11-16T09:58:17,645 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/d9e28756f78b0555a6733ffbf6ab98bd/C/6abf151a60c5402f91aa7af92dc02406, entries=150, sequenceid=415, filesize=12.0 K 2024-11-16T09:58:17,646 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~161.02 KB/164880, heapSize ~422.58 KB/432720, currentSize=46.96 KB/48090 for d9e28756f78b0555a6733ffbf6ab98bd in 511ms, sequenceid=415, compaction requested=false 2024-11-16T09:58:17,646 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for d9e28756f78b0555a6733ffbf6ab98bd: 2024-11-16T09:58:17,699 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=37267 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=18 2024-11-16T09:58:17,699 DEBUG [RS_FLUSH_OPERATIONS-regionserver/037c58a19ddb:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731751082172.d9e28756f78b0555a6733ffbf6ab98bd. 2024-11-16T09:58:17,700 INFO [RS_FLUSH_OPERATIONS-regionserver/037c58a19ddb:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HRegion(2902): Flushing d9e28756f78b0555a6733ffbf6ab98bd 3/3 column families, dataSize=46.96 KB heapSize=123.80 KB 2024-11-16T09:58:17,706 DEBUG [RS_FLUSH_OPERATIONS-regionserver/037c58a19ddb:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/d9e28756f78b0555a6733ffbf6ab98bd/.tmp/A/5218094697594d79beed890deb9c28c5 is 50, key is test_row_0/A:col10/1731751097147/Put/seqid=0 2024-11-16T09:58:17,717 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37591 is added to blk_1073741924_1100 (size=12301) 2024-11-16T09:58:17,720 INFO [RS_FLUSH_OPERATIONS-regionserver/037c58a19ddb:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=427 (bloomFilter=true), to=hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/d9e28756f78b0555a6733ffbf6ab98bd/.tmp/A/5218094697594d79beed890deb9c28c5 2024-11-16T09:58:17,732 DEBUG [RS_FLUSH_OPERATIONS-regionserver/037c58a19ddb:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/d9e28756f78b0555a6733ffbf6ab98bd/.tmp/B/35e2c5c8651a4291bbb7092d0b067dac is 50, key is test_row_0/B:col10/1731751097147/Put/seqid=0 2024-11-16T09:58:17,744 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37591 is added to blk_1073741925_1101 (size=12301) 2024-11-16T09:58:17,775 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] regionserver.HRegion(8855): Flush requested on d9e28756f78b0555a6733ffbf6ab98bd 2024-11-16T09:58:17,775 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731751082172.d9e28756f78b0555a6733ffbf6ab98bd. as already flushing 2024-11-16T09:58:17,824 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9e28756f78b0555a6733ffbf6ab98bd, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T09:58:17,824 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] ipc.CallRunner(138): callId: 124 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39696 deadline: 1731751157820, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9e28756f78b0555a6733ffbf6ab98bd, server=037c58a19ddb,37267,1731751077844 2024-11-16T09:58:17,825 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731751082172.d9e28756f78b0555a6733ffbf6ab98bd., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 , the old value is region=TestAcidGuarantees,,1731751082172.d9e28756f78b0555a6733ffbf6ab98bd., hostname=037c58a19ddb,37267,1731751077844, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9e28756f78b0555a6733ffbf6ab98bd, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:58:17,825 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731751082172.d9e28756f78b0555a6733ffbf6ab98bd., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9e28756f78b0555a6733ffbf6ab98bd, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:58:17,825 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731751082172.d9e28756f78b0555a6733ffbf6ab98bd., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 because the exception is null or not the one we care about 2024-11-16T09:58:17,829 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9e28756f78b0555a6733ffbf6ab98bd, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T09:58:17,829 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] ipc.CallRunner(138): callId: 127 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39620 deadline: 1731751157822, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9e28756f78b0555a6733ffbf6ab98bd, server=037c58a19ddb,37267,1731751077844 2024-11-16T09:58:17,829 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9e28756f78b0555a6733ffbf6ab98bd, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T09:58:17,830 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731751082172.d9e28756f78b0555a6733ffbf6ab98bd., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 , the old value is region=TestAcidGuarantees,,1731751082172.d9e28756f78b0555a6733ffbf6ab98bd., hostname=037c58a19ddb,37267,1731751077844, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9e28756f78b0555a6733ffbf6ab98bd, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:58:17,830 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731751082172.d9e28756f78b0555a6733ffbf6ab98bd., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9e28756f78b0555a6733ffbf6ab98bd, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:58:17,830 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] ipc.CallRunner(138): callId: 124 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39630 deadline: 1731751157823, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9e28756f78b0555a6733ffbf6ab98bd, server=037c58a19ddb,37267,1731751077844 2024-11-16T09:58:17,830 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731751082172.d9e28756f78b0555a6733ffbf6ab98bd., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 because the exception is null or not the one we care about 2024-11-16T09:58:17,830 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731751082172.d9e28756f78b0555a6733ffbf6ab98bd., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 , the old value is region=TestAcidGuarantees,,1731751082172.d9e28756f78b0555a6733ffbf6ab98bd., hostname=037c58a19ddb,37267,1731751077844, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9e28756f78b0555a6733ffbf6ab98bd, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:58:17,830 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731751082172.d9e28756f78b0555a6733ffbf6ab98bd., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9e28756f78b0555a6733ffbf6ab98bd, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:58:17,830 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731751082172.d9e28756f78b0555a6733ffbf6ab98bd., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 because the exception is null or not the one we care about 2024-11-16T09:58:17,830 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9e28756f78b0555a6733ffbf6ab98bd, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T09:58:17,830 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] ipc.CallRunner(138): callId: 119 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39638 deadline: 1731751157825, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9e28756f78b0555a6733ffbf6ab98bd, server=037c58a19ddb,37267,1731751077844 2024-11-16T09:58:17,831 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9e28756f78b0555a6733ffbf6ab98bd, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T09:58:17,831 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] ipc.CallRunner(138): callId: 127 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39654 deadline: 1731751157825, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9e28756f78b0555a6733ffbf6ab98bd, server=037c58a19ddb,37267,1731751077844 2024-11-16T09:58:17,831 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731751082172.d9e28756f78b0555a6733ffbf6ab98bd., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 , the old value is region=TestAcidGuarantees,,1731751082172.d9e28756f78b0555a6733ffbf6ab98bd., hostname=037c58a19ddb,37267,1731751077844, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9e28756f78b0555a6733ffbf6ab98bd, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:58:17,831 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731751082172.d9e28756f78b0555a6733ffbf6ab98bd., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9e28756f78b0555a6733ffbf6ab98bd, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:58:17,831 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731751082172.d9e28756f78b0555a6733ffbf6ab98bd., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 because the exception is null or not the one we care about 2024-11-16T09:58:17,831 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731751082172.d9e28756f78b0555a6733ffbf6ab98bd., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 , the old value is region=TestAcidGuarantees,,1731751082172.d9e28756f78b0555a6733ffbf6ab98bd., hostname=037c58a19ddb,37267,1731751077844, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9e28756f78b0555a6733ffbf6ab98bd, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:58:17,831 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731751082172.d9e28756f78b0555a6733ffbf6ab98bd., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9e28756f78b0555a6733ffbf6ab98bd, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:58:17,831 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731751082172.d9e28756f78b0555a6733ffbf6ab98bd., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 because the exception is null or not the one we care about 2024-11-16T09:58:17,863 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37877 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=17 2024-11-16T09:58:17,934 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9e28756f78b0555a6733ffbf6ab98bd, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T09:58:17,935 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] ipc.CallRunner(138): callId: 125 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39696 deadline: 1731751157933, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9e28756f78b0555a6733ffbf6ab98bd, server=037c58a19ddb,37267,1731751077844 2024-11-16T09:58:17,935 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731751082172.d9e28756f78b0555a6733ffbf6ab98bd., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 , the old value is region=TestAcidGuarantees,,1731751082172.d9e28756f78b0555a6733ffbf6ab98bd., hostname=037c58a19ddb,37267,1731751077844, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9e28756f78b0555a6733ffbf6ab98bd, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:58:17,935 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9e28756f78b0555a6733ffbf6ab98bd, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T09:58:17,935 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731751082172.d9e28756f78b0555a6733ffbf6ab98bd., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9e28756f78b0555a6733ffbf6ab98bd, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:58:17,935 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731751082172.d9e28756f78b0555a6733ffbf6ab98bd., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 because the exception is null or not the one we care about 2024-11-16T09:58:17,935 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] ipc.CallRunner(138): callId: 128 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39620 deadline: 1731751157933, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9e28756f78b0555a6733ffbf6ab98bd, server=037c58a19ddb,37267,1731751077844 2024-11-16T09:58:17,936 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731751082172.d9e28756f78b0555a6733ffbf6ab98bd., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 , the old value is region=TestAcidGuarantees,,1731751082172.d9e28756f78b0555a6733ffbf6ab98bd., hostname=037c58a19ddb,37267,1731751077844, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9e28756f78b0555a6733ffbf6ab98bd, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:58:17,935 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9e28756f78b0555a6733ffbf6ab98bd, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T09:58:17,936 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731751082172.d9e28756f78b0555a6733ffbf6ab98bd., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9e28756f78b0555a6733ffbf6ab98bd, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:58:17,936 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731751082172.d9e28756f78b0555a6733ffbf6ab98bd., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 because the exception is null or not the one we care about 2024-11-16T09:58:17,936 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] ipc.CallRunner(138): callId: 120 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39638 deadline: 1731751157934, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9e28756f78b0555a6733ffbf6ab98bd, server=037c58a19ddb,37267,1731751077844 2024-11-16T09:58:17,936 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731751082172.d9e28756f78b0555a6733ffbf6ab98bd., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 , the old value is region=TestAcidGuarantees,,1731751082172.d9e28756f78b0555a6733ffbf6ab98bd., hostname=037c58a19ddb,37267,1731751077844, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9e28756f78b0555a6733ffbf6ab98bd, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:58:17,936 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9e28756f78b0555a6733ffbf6ab98bd, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T09:58:17,936 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731751082172.d9e28756f78b0555a6733ffbf6ab98bd., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9e28756f78b0555a6733ffbf6ab98bd, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:58:17,936 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731751082172.d9e28756f78b0555a6733ffbf6ab98bd., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 because the exception is null or not the one we care about 2024-11-16T09:58:17,936 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] ipc.CallRunner(138): callId: 125 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39630 deadline: 1731751157934, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9e28756f78b0555a6733ffbf6ab98bd, server=037c58a19ddb,37267,1731751077844 2024-11-16T09:58:17,937 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731751082172.d9e28756f78b0555a6733ffbf6ab98bd., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 , the old value is region=TestAcidGuarantees,,1731751082172.d9e28756f78b0555a6733ffbf6ab98bd., hostname=037c58a19ddb,37267,1731751077844, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9e28756f78b0555a6733ffbf6ab98bd, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:58:17,937 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731751082172.d9e28756f78b0555a6733ffbf6ab98bd., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9e28756f78b0555a6733ffbf6ab98bd, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:58:17,937 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731751082172.d9e28756f78b0555a6733ffbf6ab98bd., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 because the exception is null or not the one we care about 2024-11-16T09:58:17,944 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9e28756f78b0555a6733ffbf6ab98bd, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T09:58:17,944 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] ipc.CallRunner(138): callId: 128 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39654 deadline: 1731751157943, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9e28756f78b0555a6733ffbf6ab98bd, server=037c58a19ddb,37267,1731751077844 2024-11-16T09:58:17,945 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731751082172.d9e28756f78b0555a6733ffbf6ab98bd., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 , the old value is region=TestAcidGuarantees,,1731751082172.d9e28756f78b0555a6733ffbf6ab98bd., hostname=037c58a19ddb,37267,1731751077844, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9e28756f78b0555a6733ffbf6ab98bd, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:58:17,945 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731751082172.d9e28756f78b0555a6733ffbf6ab98bd., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9e28756f78b0555a6733ffbf6ab98bd, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:58:17,945 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731751082172.d9e28756f78b0555a6733ffbf6ab98bd., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 because the exception is null or not the one we care about 2024-11-16T09:58:17,989 DEBUG [RS:0;037c58a19ddb:37267-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/d9e28756f78b0555a6733ffbf6ab98bd/.tmp/C/bafc286da4bb4b178dc9fe040d3351f4 as hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/d9e28756f78b0555a6733ffbf6ab98bd/C/bafc286da4bb4b178dc9fe040d3351f4 2024-11-16T09:58:17,997 INFO [RS:0;037c58a19ddb:37267-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in d9e28756f78b0555a6733ffbf6ab98bd/C of d9e28756f78b0555a6733ffbf6ab98bd into bafc286da4bb4b178dc9fe040d3351f4(size=12.9 K), total size for store is 24.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-16T09:58:17,997 DEBUG [RS:0;037c58a19ddb:37267-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for d9e28756f78b0555a6733ffbf6ab98bd: 2024-11-16T09:58:17,997 INFO [RS:0;037c58a19ddb:37267-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731751082172.d9e28756f78b0555a6733ffbf6ab98bd., storeName=d9e28756f78b0555a6733ffbf6ab98bd/C, priority=12, startTime=1731751097096; duration=0sec 2024-11-16T09:58:17,998 DEBUG [RS:0;037c58a19ddb:37267-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-16T09:58:17,998 DEBUG [RS:0;037c58a19ddb:37267-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: d9e28756f78b0555a6733ffbf6ab98bd:C 2024-11-16T09:58:18,145 INFO [RS_FLUSH_OPERATIONS-regionserver/037c58a19ddb:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=427 (bloomFilter=true), to=hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/d9e28756f78b0555a6733ffbf6ab98bd/.tmp/B/35e2c5c8651a4291bbb7092d0b067dac 2024-11-16T09:58:18,145 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9e28756f78b0555a6733ffbf6ab98bd, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T09:58:18,145 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] ipc.CallRunner(138): callId: 126 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39696 deadline: 1731751158143, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9e28756f78b0555a6733ffbf6ab98bd, server=037c58a19ddb,37267,1731751077844 2024-11-16T09:58:18,145 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9e28756f78b0555a6733ffbf6ab98bd, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T09:58:18,146 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731751082172.d9e28756f78b0555a6733ffbf6ab98bd., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 , the old value is region=TestAcidGuarantees,,1731751082172.d9e28756f78b0555a6733ffbf6ab98bd., hostname=037c58a19ddb,37267,1731751077844, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9e28756f78b0555a6733ffbf6ab98bd, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:58:18,146 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9e28756f78b0555a6733ffbf6ab98bd, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T09:58:18,146 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731751082172.d9e28756f78b0555a6733ffbf6ab98bd., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9e28756f78b0555a6733ffbf6ab98bd, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:58:18,146 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] ipc.CallRunner(138): callId: 129 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39620 deadline: 1731751158143, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9e28756f78b0555a6733ffbf6ab98bd, server=037c58a19ddb,37267,1731751077844 2024-11-16T09:58:18,146 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731751082172.d9e28756f78b0555a6733ffbf6ab98bd., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 because the exception is null or not the one we care about 2024-11-16T09:58:18,146 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] ipc.CallRunner(138): callId: 121 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39638 deadline: 1731751158143, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9e28756f78b0555a6733ffbf6ab98bd, server=037c58a19ddb,37267,1731751077844 2024-11-16T09:58:18,146 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731751082172.d9e28756f78b0555a6733ffbf6ab98bd., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 , the old value is region=TestAcidGuarantees,,1731751082172.d9e28756f78b0555a6733ffbf6ab98bd., hostname=037c58a19ddb,37267,1731751077844, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9e28756f78b0555a6733ffbf6ab98bd, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:58:18,146 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731751082172.d9e28756f78b0555a6733ffbf6ab98bd., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 , the old value is region=TestAcidGuarantees,,1731751082172.d9e28756f78b0555a6733ffbf6ab98bd., hostname=037c58a19ddb,37267,1731751077844, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9e28756f78b0555a6733ffbf6ab98bd, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:58:18,146 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9e28756f78b0555a6733ffbf6ab98bd, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T09:58:18,146 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731751082172.d9e28756f78b0555a6733ffbf6ab98bd., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9e28756f78b0555a6733ffbf6ab98bd, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:58:18,146 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731751082172.d9e28756f78b0555a6733ffbf6ab98bd., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9e28756f78b0555a6733ffbf6ab98bd, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:58:18,146 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731751082172.d9e28756f78b0555a6733ffbf6ab98bd., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 because the exception is null or not the one we care about 2024-11-16T09:58:18,146 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731751082172.d9e28756f78b0555a6733ffbf6ab98bd., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 because the exception is null or not the one we care about 2024-11-16T09:58:18,146 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] ipc.CallRunner(138): callId: 126 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39630 deadline: 1731751158143, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9e28756f78b0555a6733ffbf6ab98bd, server=037c58a19ddb,37267,1731751077844 2024-11-16T09:58:18,147 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731751082172.d9e28756f78b0555a6733ffbf6ab98bd., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 , the old value is region=TestAcidGuarantees,,1731751082172.d9e28756f78b0555a6733ffbf6ab98bd., hostname=037c58a19ddb,37267,1731751077844, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9e28756f78b0555a6733ffbf6ab98bd, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:58:18,147 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731751082172.d9e28756f78b0555a6733ffbf6ab98bd., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9e28756f78b0555a6733ffbf6ab98bd, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:58:18,147 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731751082172.d9e28756f78b0555a6733ffbf6ab98bd., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 because the exception is null or not the one we care about 2024-11-16T09:58:18,154 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9e28756f78b0555a6733ffbf6ab98bd, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T09:58:18,154 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] ipc.CallRunner(138): callId: 129 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39654 deadline: 1731751158153, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9e28756f78b0555a6733ffbf6ab98bd, server=037c58a19ddb,37267,1731751077844 2024-11-16T09:58:18,155 DEBUG [RS_FLUSH_OPERATIONS-regionserver/037c58a19ddb:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/d9e28756f78b0555a6733ffbf6ab98bd/.tmp/C/0f9fc7e9d0654382b9999d87e440edee is 50, key is test_row_0/C:col10/1731751097147/Put/seqid=0 2024-11-16T09:58:18,155 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731751082172.d9e28756f78b0555a6733ffbf6ab98bd., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 , the old value is region=TestAcidGuarantees,,1731751082172.d9e28756f78b0555a6733ffbf6ab98bd., hostname=037c58a19ddb,37267,1731751077844, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9e28756f78b0555a6733ffbf6ab98bd, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:58:18,155 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731751082172.d9e28756f78b0555a6733ffbf6ab98bd., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9e28756f78b0555a6733ffbf6ab98bd, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:58:18,155 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731751082172.d9e28756f78b0555a6733ffbf6ab98bd., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 because the exception is null or not the one we care about 2024-11-16T09:58:18,166 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37591 is added to blk_1073741926_1102 (size=12301) 2024-11-16T09:58:18,166 INFO [RS_FLUSH_OPERATIONS-regionserver/037c58a19ddb:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=427 (bloomFilter=true), to=hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/d9e28756f78b0555a6733ffbf6ab98bd/.tmp/C/0f9fc7e9d0654382b9999d87e440edee 2024-11-16T09:58:18,173 DEBUG [RS_FLUSH_OPERATIONS-regionserver/037c58a19ddb:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/d9e28756f78b0555a6733ffbf6ab98bd/.tmp/A/5218094697594d79beed890deb9c28c5 as hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/d9e28756f78b0555a6733ffbf6ab98bd/A/5218094697594d79beed890deb9c28c5 2024-11-16T09:58:18,179 INFO [RS_FLUSH_OPERATIONS-regionserver/037c58a19ddb:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/d9e28756f78b0555a6733ffbf6ab98bd/A/5218094697594d79beed890deb9c28c5, entries=150, sequenceid=427, filesize=12.0 K 2024-11-16T09:58:18,180 DEBUG [RS_FLUSH_OPERATIONS-regionserver/037c58a19ddb:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/d9e28756f78b0555a6733ffbf6ab98bd/.tmp/B/35e2c5c8651a4291bbb7092d0b067dac as hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/d9e28756f78b0555a6733ffbf6ab98bd/B/35e2c5c8651a4291bbb7092d0b067dac 2024-11-16T09:58:18,186 INFO [RS_FLUSH_OPERATIONS-regionserver/037c58a19ddb:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/d9e28756f78b0555a6733ffbf6ab98bd/B/35e2c5c8651a4291bbb7092d0b067dac, entries=150, sequenceid=427, filesize=12.0 K 2024-11-16T09:58:18,187 DEBUG [RS_FLUSH_OPERATIONS-regionserver/037c58a19ddb:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/d9e28756f78b0555a6733ffbf6ab98bd/.tmp/C/0f9fc7e9d0654382b9999d87e440edee as hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/d9e28756f78b0555a6733ffbf6ab98bd/C/0f9fc7e9d0654382b9999d87e440edee 2024-11-16T09:58:18,193 INFO [RS_FLUSH_OPERATIONS-regionserver/037c58a19ddb:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/d9e28756f78b0555a6733ffbf6ab98bd/C/0f9fc7e9d0654382b9999d87e440edee, entries=150, sequenceid=427, filesize=12.0 K 2024-11-16T09:58:18,194 INFO [RS_FLUSH_OPERATIONS-regionserver/037c58a19ddb:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HRegion(3140): Finished flush of dataSize ~46.96 KB/48090, heapSize ~123.75 KB/126720, currentSize=161.02 KB/164880 for d9e28756f78b0555a6733ffbf6ab98bd in 494ms, sequenceid=427, compaction requested=true 2024-11-16T09:58:18,195 DEBUG [RS_FLUSH_OPERATIONS-regionserver/037c58a19ddb:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HRegion(2603): Flush status journal for d9e28756f78b0555a6733ffbf6ab98bd: 2024-11-16T09:58:18,195 DEBUG [RS_FLUSH_OPERATIONS-regionserver/037c58a19ddb:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731751082172.d9e28756f78b0555a6733ffbf6ab98bd. 2024-11-16T09:58:18,195 DEBUG [RS_FLUSH_OPERATIONS-regionserver/037c58a19ddb:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=18 2024-11-16T09:58:18,195 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37877 {}] master.HMaster(4169): Remote procedure done, pid=18 2024-11-16T09:58:18,198 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=18, resume processing ppid=17 2024-11-16T09:58:18,199 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=18, ppid=17, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 960 msec 2024-11-16T09:58:18,200 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=17, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=17, table=TestAcidGuarantees in 966 msec 2024-11-16T09:58:18,373 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37877 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=17 2024-11-16T09:58:18,373 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-16T09:58:18,374 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37877 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-16T09:58:18,375 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37877 {}] procedure2.ProcedureExecutor(1139): Stored pid=19, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=19, table=TestAcidGuarantees 2024-11-16T09:58:18,376 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37877 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=19 2024-11-16T09:58:18,376 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=19, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=19, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-16T09:58:18,377 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=19, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=19, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-16T09:58:18,378 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=20, ppid=19, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-16T09:58:18,456 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] regionserver.HRegion(8855): Flush requested on d9e28756f78b0555a6733ffbf6ab98bd 2024-11-16T09:58:18,456 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-16T09:58:18,456 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing d9e28756f78b0555a6733ffbf6ab98bd 3/3 column families, dataSize=167.72 KB heapSize=440.20 KB 2024-11-16T09:58:18,463 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/d9e28756f78b0555a6733ffbf6ab98bd/.tmp/A/6ba55af74e864857b6100bf2a5217c3d is 50, key is test_row_0/A:col10/1731751097823/Put/seqid=0 2024-11-16T09:58:18,464 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9e28756f78b0555a6733ffbf6ab98bd, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T09:58:18,464 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] ipc.CallRunner(138): callId: 128 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39630 deadline: 1731751158460, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9e28756f78b0555a6733ffbf6ab98bd, server=037c58a19ddb,37267,1731751077844 2024-11-16T09:58:18,464 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731751082172.d9e28756f78b0555a6733ffbf6ab98bd., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 , the old value is region=TestAcidGuarantees,,1731751082172.d9e28756f78b0555a6733ffbf6ab98bd., hostname=037c58a19ddb,37267,1731751077844, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9e28756f78b0555a6733ffbf6ab98bd, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:58:18,465 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731751082172.d9e28756f78b0555a6733ffbf6ab98bd., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9e28756f78b0555a6733ffbf6ab98bd, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:58:18,465 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731751082172.d9e28756f78b0555a6733ffbf6ab98bd., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 because the exception is null or not the one we care about 2024-11-16T09:58:18,465 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9e28756f78b0555a6733ffbf6ab98bd, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T09:58:18,465 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] ipc.CallRunner(138): callId: 131 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39620 deadline: 1731751158460, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9e28756f78b0555a6733ffbf6ab98bd, server=037c58a19ddb,37267,1731751077844 2024-11-16T09:58:18,465 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731751082172.d9e28756f78b0555a6733ffbf6ab98bd., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 , the old value is region=TestAcidGuarantees,,1731751082172.d9e28756f78b0555a6733ffbf6ab98bd., hostname=037c58a19ddb,37267,1731751077844, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9e28756f78b0555a6733ffbf6ab98bd, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:58:18,465 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731751082172.d9e28756f78b0555a6733ffbf6ab98bd., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9e28756f78b0555a6733ffbf6ab98bd, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:58:18,466 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731751082172.d9e28756f78b0555a6733ffbf6ab98bd., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 because the exception is null or not the one we care about 2024-11-16T09:58:18,465 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9e28756f78b0555a6733ffbf6ab98bd, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T09:58:18,466 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] ipc.CallRunner(138): callId: 129 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39696 deadline: 1731751158461, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9e28756f78b0555a6733ffbf6ab98bd, server=037c58a19ddb,37267,1731751077844 2024-11-16T09:58:18,466 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731751082172.d9e28756f78b0555a6733ffbf6ab98bd., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 , the old value is region=TestAcidGuarantees,,1731751082172.d9e28756f78b0555a6733ffbf6ab98bd., hostname=037c58a19ddb,37267,1731751077844, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9e28756f78b0555a6733ffbf6ab98bd, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:58:18,466 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731751082172.d9e28756f78b0555a6733ffbf6ab98bd., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9e28756f78b0555a6733ffbf6ab98bd, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:58:18,466 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731751082172.d9e28756f78b0555a6733ffbf6ab98bd., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 because the exception is null or not the one we care about 2024-11-16T09:58:18,466 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9e28756f78b0555a6733ffbf6ab98bd, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T09:58:18,466 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] ipc.CallRunner(138): callId: 130 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39654 deadline: 1731751158463, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9e28756f78b0555a6733ffbf6ab98bd, server=037c58a19ddb,37267,1731751077844 2024-11-16T09:58:18,467 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731751082172.d9e28756f78b0555a6733ffbf6ab98bd., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 , the old value is region=TestAcidGuarantees,,1731751082172.d9e28756f78b0555a6733ffbf6ab98bd., hostname=037c58a19ddb,37267,1731751077844, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9e28756f78b0555a6733ffbf6ab98bd, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:58:18,467 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731751082172.d9e28756f78b0555a6733ffbf6ab98bd., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9e28756f78b0555a6733ffbf6ab98bd, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:58:18,467 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731751082172.d9e28756f78b0555a6733ffbf6ab98bd., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 because the exception is null or not the one we care about 2024-11-16T09:58:18,467 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9e28756f78b0555a6733ffbf6ab98bd, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T09:58:18,467 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] ipc.CallRunner(138): callId: 124 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39638 deadline: 1731751158465, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9e28756f78b0555a6733ffbf6ab98bd, server=037c58a19ddb,37267,1731751077844 2024-11-16T09:58:18,467 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731751082172.d9e28756f78b0555a6733ffbf6ab98bd., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 , the old value is region=TestAcidGuarantees,,1731751082172.d9e28756f78b0555a6733ffbf6ab98bd., hostname=037c58a19ddb,37267,1731751077844, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9e28756f78b0555a6733ffbf6ab98bd, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:58:18,468 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731751082172.d9e28756f78b0555a6733ffbf6ab98bd., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9e28756f78b0555a6733ffbf6ab98bd, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:58:18,468 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731751082172.d9e28756f78b0555a6733ffbf6ab98bd., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 because the exception is null or not the one we care about 2024-11-16T09:58:18,478 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37591 is added to blk_1073741927_1103 (size=12301) 2024-11-16T09:58:18,483 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37877 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=19 2024-11-16T09:58:18,530 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=37267 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=20 2024-11-16T09:58:18,530 DEBUG [RS_FLUSH_OPERATIONS-regionserver/037c58a19ddb:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731751082172.d9e28756f78b0555a6733ffbf6ab98bd. 2024-11-16T09:58:18,530 DEBUG [RS_FLUSH_OPERATIONS-regionserver/037c58a19ddb:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731751082172.d9e28756f78b0555a6733ffbf6ab98bd. as already flushing 2024-11-16T09:58:18,531 DEBUG [RS_FLUSH_OPERATIONS-regionserver/037c58a19ddb:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731751082172.d9e28756f78b0555a6733ffbf6ab98bd. 2024-11-16T09:58:18,531 ERROR [RS_FLUSH_OPERATIONS-regionserver/037c58a19ddb:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] handler.RSProcedureHandler(58): pid=20 java.io.IOException: Unable to complete flush {ENCODED => d9e28756f78b0555a6733ffbf6ab98bd, NAME => 'TestAcidGuarantees,,1731751082172.d9e28756f78b0555a6733ffbf6ab98bd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T09:58:18,531 DEBUG [RS_FLUSH_OPERATIONS-regionserver/037c58a19ddb:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=20 java.io.IOException: Unable to complete flush {ENCODED => d9e28756f78b0555a6733ffbf6ab98bd, NAME => 'TestAcidGuarantees,,1731751082172.d9e28756f78b0555a6733ffbf6ab98bd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T09:58:18,532 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37877 {}] master.HMaster(4177): Remote procedure failed, pid=20 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d9e28756f78b0555a6733ffbf6ab98bd, NAME => 'TestAcidGuarantees,,1731751082172.d9e28756f78b0555a6733ffbf6ab98bd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d9e28756f78b0555a6733ffbf6ab98bd, NAME => 'TestAcidGuarantees,,1731751082172.d9e28756f78b0555a6733ffbf6ab98bd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T09:58:18,574 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9e28756f78b0555a6733ffbf6ab98bd, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T09:58:18,574 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] ipc.CallRunner(138): callId: 129 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39630 deadline: 1731751158573, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9e28756f78b0555a6733ffbf6ab98bd, server=037c58a19ddb,37267,1731751077844 2024-11-16T09:58:18,575 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731751082172.d9e28756f78b0555a6733ffbf6ab98bd., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 , the old value is region=TestAcidGuarantees,,1731751082172.d9e28756f78b0555a6733ffbf6ab98bd., hostname=037c58a19ddb,37267,1731751077844, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9e28756f78b0555a6733ffbf6ab98bd, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:58:18,575 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731751082172.d9e28756f78b0555a6733ffbf6ab98bd., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9e28756f78b0555a6733ffbf6ab98bd, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:58:18,575 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731751082172.d9e28756f78b0555a6733ffbf6ab98bd., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 because the exception is null or not the one we care about 2024-11-16T09:58:18,575 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9e28756f78b0555a6733ffbf6ab98bd, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T09:58:18,575 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9e28756f78b0555a6733ffbf6ab98bd, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T09:58:18,575 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] ipc.CallRunner(138): callId: 132 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39620 deadline: 1731751158574, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9e28756f78b0555a6733ffbf6ab98bd, server=037c58a19ddb,37267,1731751077844 2024-11-16T09:58:18,575 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] ipc.CallRunner(138): callId: 130 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39696 deadline: 1731751158574, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9e28756f78b0555a6733ffbf6ab98bd, server=037c58a19ddb,37267,1731751077844 2024-11-16T09:58:18,575 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731751082172.d9e28756f78b0555a6733ffbf6ab98bd., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 , the old value is region=TestAcidGuarantees,,1731751082172.d9e28756f78b0555a6733ffbf6ab98bd., hostname=037c58a19ddb,37267,1731751077844, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9e28756f78b0555a6733ffbf6ab98bd, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:58:18,575 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9e28756f78b0555a6733ffbf6ab98bd, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T09:58:18,575 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731751082172.d9e28756f78b0555a6733ffbf6ab98bd., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9e28756f78b0555a6733ffbf6ab98bd, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:58:18,575 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731751082172.d9e28756f78b0555a6733ffbf6ab98bd., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 because the exception is null or not the one we care about 2024-11-16T09:58:18,576 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] ipc.CallRunner(138): callId: 125 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39638 deadline: 1731751158574, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9e28756f78b0555a6733ffbf6ab98bd, server=037c58a19ddb,37267,1731751077844 2024-11-16T09:58:18,576 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731751082172.d9e28756f78b0555a6733ffbf6ab98bd., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 , the old value is region=TestAcidGuarantees,,1731751082172.d9e28756f78b0555a6733ffbf6ab98bd., hostname=037c58a19ddb,37267,1731751077844, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9e28756f78b0555a6733ffbf6ab98bd, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:58:18,576 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731751082172.d9e28756f78b0555a6733ffbf6ab98bd., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9e28756f78b0555a6733ffbf6ab98bd, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:58:18,576 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731751082172.d9e28756f78b0555a6733ffbf6ab98bd., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 because the exception is null or not the one we care about 2024-11-16T09:58:18,576 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731751082172.d9e28756f78b0555a6733ffbf6ab98bd., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 , the old value is region=TestAcidGuarantees,,1731751082172.d9e28756f78b0555a6733ffbf6ab98bd., hostname=037c58a19ddb,37267,1731751077844, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9e28756f78b0555a6733ffbf6ab98bd, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:58:18,576 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731751082172.d9e28756f78b0555a6733ffbf6ab98bd., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9e28756f78b0555a6733ffbf6ab98bd, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:58:18,576 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731751082172.d9e28756f78b0555a6733ffbf6ab98bd., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 because the exception is null or not the one we care about 2024-11-16T09:58:18,684 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=37267 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=20 2024-11-16T09:58:18,684 DEBUG [RS_FLUSH_OPERATIONS-regionserver/037c58a19ddb:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731751082172.d9e28756f78b0555a6733ffbf6ab98bd. 2024-11-16T09:58:18,684 DEBUG [RS_FLUSH_OPERATIONS-regionserver/037c58a19ddb:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731751082172.d9e28756f78b0555a6733ffbf6ab98bd. as already flushing 2024-11-16T09:58:18,685 DEBUG [RS_FLUSH_OPERATIONS-regionserver/037c58a19ddb:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731751082172.d9e28756f78b0555a6733ffbf6ab98bd. 2024-11-16T09:58:18,685 ERROR [RS_FLUSH_OPERATIONS-regionserver/037c58a19ddb:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] handler.RSProcedureHandler(58): pid=20 java.io.IOException: Unable to complete flush {ENCODED => d9e28756f78b0555a6733ffbf6ab98bd, NAME => 'TestAcidGuarantees,,1731751082172.d9e28756f78b0555a6733ffbf6ab98bd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T09:58:18,685 DEBUG [RS_FLUSH_OPERATIONS-regionserver/037c58a19ddb:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=20 java.io.IOException: Unable to complete flush {ENCODED => d9e28756f78b0555a6733ffbf6ab98bd, NAME => 'TestAcidGuarantees,,1731751082172.d9e28756f78b0555a6733ffbf6ab98bd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T09:58:18,686 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37877 {}] master.HMaster(4177): Remote procedure failed, pid=20 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d9e28756f78b0555a6733ffbf6ab98bd, NAME => 'TestAcidGuarantees,,1731751082172.d9e28756f78b0555a6733ffbf6ab98bd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d9e28756f78b0555a6733ffbf6ab98bd, NAME => 'TestAcidGuarantees,,1731751082172.d9e28756f78b0555a6733ffbf6ab98bd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T09:58:18,693 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37877 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=19 2024-11-16T09:58:18,784 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9e28756f78b0555a6733ffbf6ab98bd, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T09:58:18,784 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] ipc.CallRunner(138): callId: 130 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39630 deadline: 1731751158782, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9e28756f78b0555a6733ffbf6ab98bd, server=037c58a19ddb,37267,1731751077844 2024-11-16T09:58:18,784 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731751082172.d9e28756f78b0555a6733ffbf6ab98bd., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 , the old value is region=TestAcidGuarantees,,1731751082172.d9e28756f78b0555a6733ffbf6ab98bd., hostname=037c58a19ddb,37267,1731751077844, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9e28756f78b0555a6733ffbf6ab98bd, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:58:18,785 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731751082172.d9e28756f78b0555a6733ffbf6ab98bd., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9e28756f78b0555a6733ffbf6ab98bd, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:58:18,784 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9e28756f78b0555a6733ffbf6ab98bd, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T09:58:18,785 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731751082172.d9e28756f78b0555a6733ffbf6ab98bd., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 because the exception is null or not the one we care about 2024-11-16T09:58:18,785 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] ipc.CallRunner(138): callId: 133 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39620 deadline: 1731751158783, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9e28756f78b0555a6733ffbf6ab98bd, server=037c58a19ddb,37267,1731751077844 2024-11-16T09:58:18,785 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9e28756f78b0555a6733ffbf6ab98bd, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T09:58:18,785 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731751082172.d9e28756f78b0555a6733ffbf6ab98bd., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 , the old value is region=TestAcidGuarantees,,1731751082172.d9e28756f78b0555a6733ffbf6ab98bd., hostname=037c58a19ddb,37267,1731751077844, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9e28756f78b0555a6733ffbf6ab98bd, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:58:18,785 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731751082172.d9e28756f78b0555a6733ffbf6ab98bd., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9e28756f78b0555a6733ffbf6ab98bd, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:58:18,785 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] ipc.CallRunner(138): callId: 131 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39696 deadline: 1731751158783, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9e28756f78b0555a6733ffbf6ab98bd, server=037c58a19ddb,37267,1731751077844 2024-11-16T09:58:18,785 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731751082172.d9e28756f78b0555a6733ffbf6ab98bd., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 because the exception is null or not the one we care about 2024-11-16T09:58:18,786 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731751082172.d9e28756f78b0555a6733ffbf6ab98bd., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 , the old value is region=TestAcidGuarantees,,1731751082172.d9e28756f78b0555a6733ffbf6ab98bd., hostname=037c58a19ddb,37267,1731751077844, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9e28756f78b0555a6733ffbf6ab98bd, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:58:18,786 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731751082172.d9e28756f78b0555a6733ffbf6ab98bd., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9e28756f78b0555a6733ffbf6ab98bd, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:58:18,786 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9e28756f78b0555a6733ffbf6ab98bd, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T09:58:18,786 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731751082172.d9e28756f78b0555a6733ffbf6ab98bd., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 because the exception is null or not the one we care about 2024-11-16T09:58:18,786 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] ipc.CallRunner(138): callId: 126 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39638 deadline: 1731751158783, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9e28756f78b0555a6733ffbf6ab98bd, server=037c58a19ddb,37267,1731751077844 2024-11-16T09:58:18,786 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731751082172.d9e28756f78b0555a6733ffbf6ab98bd., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 , the old value is region=TestAcidGuarantees,,1731751082172.d9e28756f78b0555a6733ffbf6ab98bd., hostname=037c58a19ddb,37267,1731751077844, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9e28756f78b0555a6733ffbf6ab98bd, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:58:18,786 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731751082172.d9e28756f78b0555a6733ffbf6ab98bd., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9e28756f78b0555a6733ffbf6ab98bd, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:58:18,786 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731751082172.d9e28756f78b0555a6733ffbf6ab98bd., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 because the exception is null or not the one we care about 2024-11-16T09:58:18,838 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=37267 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=20 2024-11-16T09:58:18,838 DEBUG [RS_FLUSH_OPERATIONS-regionserver/037c58a19ddb:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731751082172.d9e28756f78b0555a6733ffbf6ab98bd. 2024-11-16T09:58:18,838 DEBUG [RS_FLUSH_OPERATIONS-regionserver/037c58a19ddb:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731751082172.d9e28756f78b0555a6733ffbf6ab98bd. as already flushing 2024-11-16T09:58:18,838 DEBUG [RS_FLUSH_OPERATIONS-regionserver/037c58a19ddb:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731751082172.d9e28756f78b0555a6733ffbf6ab98bd. 2024-11-16T09:58:18,838 ERROR [RS_FLUSH_OPERATIONS-regionserver/037c58a19ddb:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] handler.RSProcedureHandler(58): pid=20 java.io.IOException: Unable to complete flush {ENCODED => d9e28756f78b0555a6733ffbf6ab98bd, NAME => 'TestAcidGuarantees,,1731751082172.d9e28756f78b0555a6733ffbf6ab98bd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T09:58:18,838 DEBUG [RS_FLUSH_OPERATIONS-regionserver/037c58a19ddb:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=20 java.io.IOException: Unable to complete flush {ENCODED => d9e28756f78b0555a6733ffbf6ab98bd, NAME => 'TestAcidGuarantees,,1731751082172.d9e28756f78b0555a6733ffbf6ab98bd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T09:58:18,839 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37877 {}] master.HMaster(4177): Remote procedure failed, pid=20 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d9e28756f78b0555a6733ffbf6ab98bd, NAME => 'TestAcidGuarantees,,1731751082172.d9e28756f78b0555a6733ffbf6ab98bd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d9e28756f78b0555a6733ffbf6ab98bd, NAME => 'TestAcidGuarantees,,1731751082172.d9e28756f78b0555a6733ffbf6ab98bd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T09:58:18,880 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=55.91 KB at sequenceid=456 (bloomFilter=true), to=hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/d9e28756f78b0555a6733ffbf6ab98bd/.tmp/A/6ba55af74e864857b6100bf2a5217c3d 2024-11-16T09:58:18,889 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/d9e28756f78b0555a6733ffbf6ab98bd/.tmp/B/7c5a81154f154146ac304195f673f3de is 50, key is test_row_0/B:col10/1731751097823/Put/seqid=0 2024-11-16T09:58:18,893 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37591 is added to blk_1073741928_1104 (size=12301) 2024-11-16T09:58:18,974 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9e28756f78b0555a6733ffbf6ab98bd, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T09:58:18,975 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] ipc.CallRunner(138): callId: 131 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39654 deadline: 1731751158973, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9e28756f78b0555a6733ffbf6ab98bd, server=037c58a19ddb,37267,1731751077844 2024-11-16T09:58:18,975 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731751082172.d9e28756f78b0555a6733ffbf6ab98bd., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 , the old value is region=TestAcidGuarantees,,1731751082172.d9e28756f78b0555a6733ffbf6ab98bd., hostname=037c58a19ddb,37267,1731751077844, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9e28756f78b0555a6733ffbf6ab98bd, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:58:18,975 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731751082172.d9e28756f78b0555a6733ffbf6ab98bd., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9e28756f78b0555a6733ffbf6ab98bd, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:58:18,975 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731751082172.d9e28756f78b0555a6733ffbf6ab98bd., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 because the exception is null or not the one we care about 2024-11-16T09:58:18,991 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=37267 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=20 2024-11-16T09:58:18,992 DEBUG [RS_FLUSH_OPERATIONS-regionserver/037c58a19ddb:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731751082172.d9e28756f78b0555a6733ffbf6ab98bd. 2024-11-16T09:58:18,992 DEBUG [RS_FLUSH_OPERATIONS-regionserver/037c58a19ddb:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731751082172.d9e28756f78b0555a6733ffbf6ab98bd. as already flushing 2024-11-16T09:58:18,992 DEBUG [RS_FLUSH_OPERATIONS-regionserver/037c58a19ddb:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731751082172.d9e28756f78b0555a6733ffbf6ab98bd. 2024-11-16T09:58:18,992 ERROR [RS_FLUSH_OPERATIONS-regionserver/037c58a19ddb:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] handler.RSProcedureHandler(58): pid=20 java.io.IOException: Unable to complete flush {ENCODED => d9e28756f78b0555a6733ffbf6ab98bd, NAME => 'TestAcidGuarantees,,1731751082172.d9e28756f78b0555a6733ffbf6ab98bd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T09:58:18,992 DEBUG [RS_FLUSH_OPERATIONS-regionserver/037c58a19ddb:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=20 java.io.IOException: Unable to complete flush {ENCODED => d9e28756f78b0555a6733ffbf6ab98bd, NAME => 'TestAcidGuarantees,,1731751082172.d9e28756f78b0555a6733ffbf6ab98bd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T09:58:18,993 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37877 {}] master.HMaster(4177): Remote procedure failed, pid=20 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d9e28756f78b0555a6733ffbf6ab98bd, NAME => 'TestAcidGuarantees,,1731751082172.d9e28756f78b0555a6733ffbf6ab98bd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d9e28756f78b0555a6733ffbf6ab98bd, NAME => 'TestAcidGuarantees,,1731751082172.d9e28756f78b0555a6733ffbf6ab98bd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T09:58:19,002 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37877 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=19 2024-11-16T09:58:19,095 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9e28756f78b0555a6733ffbf6ab98bd, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T09:58:19,096 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] ipc.CallRunner(138): callId: 131 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39630 deadline: 1731751159093, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9e28756f78b0555a6733ffbf6ab98bd, server=037c58a19ddb,37267,1731751077844 2024-11-16T09:58:19,096 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9e28756f78b0555a6733ffbf6ab98bd, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T09:58:19,096 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731751082172.d9e28756f78b0555a6733ffbf6ab98bd., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 , the old value is region=TestAcidGuarantees,,1731751082172.d9e28756f78b0555a6733ffbf6ab98bd., hostname=037c58a19ddb,37267,1731751077844, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9e28756f78b0555a6733ffbf6ab98bd, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:58:19,096 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731751082172.d9e28756f78b0555a6733ffbf6ab98bd., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9e28756f78b0555a6733ffbf6ab98bd, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:58:19,096 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] ipc.CallRunner(138): callId: 134 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39620 deadline: 1731751159094, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9e28756f78b0555a6733ffbf6ab98bd, server=037c58a19ddb,37267,1731751077844 2024-11-16T09:58:19,097 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731751082172.d9e28756f78b0555a6733ffbf6ab98bd., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 because the exception is null or not the one we care about 2024-11-16T09:58:19,097 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9e28756f78b0555a6733ffbf6ab98bd, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T09:58:19,097 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731751082172.d9e28756f78b0555a6733ffbf6ab98bd., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 , the old value is region=TestAcidGuarantees,,1731751082172.d9e28756f78b0555a6733ffbf6ab98bd., hostname=037c58a19ddb,37267,1731751077844, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9e28756f78b0555a6733ffbf6ab98bd, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:58:19,097 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] ipc.CallRunner(138): callId: 132 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39696 deadline: 1731751159094, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9e28756f78b0555a6733ffbf6ab98bd, server=037c58a19ddb,37267,1731751077844 2024-11-16T09:58:19,097 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731751082172.d9e28756f78b0555a6733ffbf6ab98bd., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9e28756f78b0555a6733ffbf6ab98bd, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:58:19,097 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731751082172.d9e28756f78b0555a6733ffbf6ab98bd., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 because the exception is null or not the one we care about 2024-11-16T09:58:19,097 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9e28756f78b0555a6733ffbf6ab98bd, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T09:58:19,097 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731751082172.d9e28756f78b0555a6733ffbf6ab98bd., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 , the old value is region=TestAcidGuarantees,,1731751082172.d9e28756f78b0555a6733ffbf6ab98bd., hostname=037c58a19ddb,37267,1731751077844, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9e28756f78b0555a6733ffbf6ab98bd, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:58:19,097 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731751082172.d9e28756f78b0555a6733ffbf6ab98bd., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9e28756f78b0555a6733ffbf6ab98bd, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:58:19,097 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731751082172.d9e28756f78b0555a6733ffbf6ab98bd., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 because the exception is null or not the one we care about 2024-11-16T09:58:19,097 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] ipc.CallRunner(138): callId: 127 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39638 deadline: 1731751159094, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9e28756f78b0555a6733ffbf6ab98bd, server=037c58a19ddb,37267,1731751077844 2024-11-16T09:58:19,098 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731751082172.d9e28756f78b0555a6733ffbf6ab98bd., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 , the old value is region=TestAcidGuarantees,,1731751082172.d9e28756f78b0555a6733ffbf6ab98bd., hostname=037c58a19ddb,37267,1731751077844, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9e28756f78b0555a6733ffbf6ab98bd, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:58:19,098 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731751082172.d9e28756f78b0555a6733ffbf6ab98bd., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9e28756f78b0555a6733ffbf6ab98bd, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:58:19,098 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731751082172.d9e28756f78b0555a6733ffbf6ab98bd., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 because the exception is null or not the one we care about 2024-11-16T09:58:19,145 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=37267 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=20 2024-11-16T09:58:19,146 DEBUG [RS_FLUSH_OPERATIONS-regionserver/037c58a19ddb:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731751082172.d9e28756f78b0555a6733ffbf6ab98bd. 2024-11-16T09:58:19,146 DEBUG [RS_FLUSH_OPERATIONS-regionserver/037c58a19ddb:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731751082172.d9e28756f78b0555a6733ffbf6ab98bd. as already flushing 2024-11-16T09:58:19,146 DEBUG [RS_FLUSH_OPERATIONS-regionserver/037c58a19ddb:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731751082172.d9e28756f78b0555a6733ffbf6ab98bd. 2024-11-16T09:58:19,146 ERROR [RS_FLUSH_OPERATIONS-regionserver/037c58a19ddb:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] handler.RSProcedureHandler(58): pid=20 java.io.IOException: Unable to complete flush {ENCODED => d9e28756f78b0555a6733ffbf6ab98bd, NAME => 'TestAcidGuarantees,,1731751082172.d9e28756f78b0555a6733ffbf6ab98bd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T09:58:19,146 DEBUG [RS_FLUSH_OPERATIONS-regionserver/037c58a19ddb:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=20 java.io.IOException: Unable to complete flush {ENCODED => d9e28756f78b0555a6733ffbf6ab98bd, NAME => 'TestAcidGuarantees,,1731751082172.d9e28756f78b0555a6733ffbf6ab98bd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T09:58:19,147 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37877 {}] master.HMaster(4177): Remote procedure failed, pid=20 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d9e28756f78b0555a6733ffbf6ab98bd, NAME => 'TestAcidGuarantees,,1731751082172.d9e28756f78b0555a6733ffbf6ab98bd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d9e28756f78b0555a6733ffbf6ab98bd, NAME => 'TestAcidGuarantees,,1731751082172.d9e28756f78b0555a6733ffbf6ab98bd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T09:58:19,294 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=55.91 KB at sequenceid=456 (bloomFilter=true), to=hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/d9e28756f78b0555a6733ffbf6ab98bd/.tmp/B/7c5a81154f154146ac304195f673f3de 2024-11-16T09:58:19,300 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=37267 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=20 2024-11-16T09:58:19,300 DEBUG [RS_FLUSH_OPERATIONS-regionserver/037c58a19ddb:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731751082172.d9e28756f78b0555a6733ffbf6ab98bd. 2024-11-16T09:58:19,300 DEBUG [RS_FLUSH_OPERATIONS-regionserver/037c58a19ddb:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731751082172.d9e28756f78b0555a6733ffbf6ab98bd. as already flushing 2024-11-16T09:58:19,300 DEBUG [RS_FLUSH_OPERATIONS-regionserver/037c58a19ddb:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731751082172.d9e28756f78b0555a6733ffbf6ab98bd. 2024-11-16T09:58:19,300 ERROR [RS_FLUSH_OPERATIONS-regionserver/037c58a19ddb:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] handler.RSProcedureHandler(58): pid=20 java.io.IOException: Unable to complete flush {ENCODED => d9e28756f78b0555a6733ffbf6ab98bd, NAME => 'TestAcidGuarantees,,1731751082172.d9e28756f78b0555a6733ffbf6ab98bd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T09:58:19,300 DEBUG [RS_FLUSH_OPERATIONS-regionserver/037c58a19ddb:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=20 java.io.IOException: Unable to complete flush {ENCODED => d9e28756f78b0555a6733ffbf6ab98bd, NAME => 'TestAcidGuarantees,,1731751082172.d9e28756f78b0555a6733ffbf6ab98bd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T09:58:19,301 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37877 {}] master.HMaster(4177): Remote procedure failed, pid=20 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d9e28756f78b0555a6733ffbf6ab98bd, NAME => 'TestAcidGuarantees,,1731751082172.d9e28756f78b0555a6733ffbf6ab98bd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d9e28756f78b0555a6733ffbf6ab98bd, NAME => 'TestAcidGuarantees,,1731751082172.d9e28756f78b0555a6733ffbf6ab98bd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T09:58:19,306 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/d9e28756f78b0555a6733ffbf6ab98bd/.tmp/C/191b9d45ffa9453395b379bdb72884b4 is 50, key is test_row_0/C:col10/1731751097823/Put/seqid=0 2024-11-16T09:58:19,312 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37591 is added to blk_1073741929_1105 (size=12301) 2024-11-16T09:58:19,453 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=37267 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=20 2024-11-16T09:58:19,454 DEBUG [RS_FLUSH_OPERATIONS-regionserver/037c58a19ddb:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731751082172.d9e28756f78b0555a6733ffbf6ab98bd. 2024-11-16T09:58:19,454 DEBUG [RS_FLUSH_OPERATIONS-regionserver/037c58a19ddb:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731751082172.d9e28756f78b0555a6733ffbf6ab98bd. as already flushing 2024-11-16T09:58:19,454 DEBUG [RS_FLUSH_OPERATIONS-regionserver/037c58a19ddb:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731751082172.d9e28756f78b0555a6733ffbf6ab98bd. 2024-11-16T09:58:19,454 ERROR [RS_FLUSH_OPERATIONS-regionserver/037c58a19ddb:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] handler.RSProcedureHandler(58): pid=20 java.io.IOException: Unable to complete flush {ENCODED => d9e28756f78b0555a6733ffbf6ab98bd, NAME => 'TestAcidGuarantees,,1731751082172.d9e28756f78b0555a6733ffbf6ab98bd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T09:58:19,454 DEBUG [RS_FLUSH_OPERATIONS-regionserver/037c58a19ddb:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=20 java.io.IOException: Unable to complete flush {ENCODED => d9e28756f78b0555a6733ffbf6ab98bd, NAME => 'TestAcidGuarantees,,1731751082172.d9e28756f78b0555a6733ffbf6ab98bd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T09:58:19,455 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37877 {}] master.HMaster(4177): Remote procedure failed, pid=20 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d9e28756f78b0555a6733ffbf6ab98bd, NAME => 'TestAcidGuarantees,,1731751082172.d9e28756f78b0555a6733ffbf6ab98bd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d9e28756f78b0555a6733ffbf6ab98bd, NAME => 'TestAcidGuarantees,,1731751082172.d9e28756f78b0555a6733ffbf6ab98bd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T09:58:19,513 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37877 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=19 2024-11-16T09:58:19,604 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9e28756f78b0555a6733ffbf6ab98bd, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T09:58:19,604 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] ipc.CallRunner(138): callId: 135 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39620 deadline: 1731751159603, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9e28756f78b0555a6733ffbf6ab98bd, server=037c58a19ddb,37267,1731751077844 2024-11-16T09:58:19,604 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9e28756f78b0555a6733ffbf6ab98bd, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T09:58:19,605 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] ipc.CallRunner(138): callId: 132 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39630 deadline: 1731751159603, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9e28756f78b0555a6733ffbf6ab98bd, server=037c58a19ddb,37267,1731751077844 2024-11-16T09:58:19,605 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731751082172.d9e28756f78b0555a6733ffbf6ab98bd., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 , the old value is region=TestAcidGuarantees,,1731751082172.d9e28756f78b0555a6733ffbf6ab98bd., hostname=037c58a19ddb,37267,1731751077844, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9e28756f78b0555a6733ffbf6ab98bd, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:58:19,605 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9e28756f78b0555a6733ffbf6ab98bd, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T09:58:19,605 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731751082172.d9e28756f78b0555a6733ffbf6ab98bd., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9e28756f78b0555a6733ffbf6ab98bd, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:58:19,605 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731751082172.d9e28756f78b0555a6733ffbf6ab98bd., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 because the exception is null or not the one we care about 2024-11-16T09:58:19,605 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9e28756f78b0555a6733ffbf6ab98bd, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T09:58:19,605 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] ipc.CallRunner(138): callId: 133 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39696 deadline: 1731751159603, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9e28756f78b0555a6733ffbf6ab98bd, server=037c58a19ddb,37267,1731751077844 2024-11-16T09:58:19,605 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] ipc.CallRunner(138): callId: 128 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39638 deadline: 1731751159603, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9e28756f78b0555a6733ffbf6ab98bd, server=037c58a19ddb,37267,1731751077844 2024-11-16T09:58:19,605 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731751082172.d9e28756f78b0555a6733ffbf6ab98bd., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 , the old value is region=TestAcidGuarantees,,1731751082172.d9e28756f78b0555a6733ffbf6ab98bd., hostname=037c58a19ddb,37267,1731751077844, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9e28756f78b0555a6733ffbf6ab98bd, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:58:19,605 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731751082172.d9e28756f78b0555a6733ffbf6ab98bd., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9e28756f78b0555a6733ffbf6ab98bd, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:58:19,605 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731751082172.d9e28756f78b0555a6733ffbf6ab98bd., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 because the exception is null or not the one we care about 2024-11-16T09:58:19,606 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731751082172.d9e28756f78b0555a6733ffbf6ab98bd., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 , the old value is region=TestAcidGuarantees,,1731751082172.d9e28756f78b0555a6733ffbf6ab98bd., hostname=037c58a19ddb,37267,1731751077844, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9e28756f78b0555a6733ffbf6ab98bd, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:58:19,606 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731751082172.d9e28756f78b0555a6733ffbf6ab98bd., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9e28756f78b0555a6733ffbf6ab98bd, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:58:19,606 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731751082172.d9e28756f78b0555a6733ffbf6ab98bd., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 because the exception is null or not the one we care about 2024-11-16T09:58:19,606 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731751082172.d9e28756f78b0555a6733ffbf6ab98bd., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 , the old value is region=TestAcidGuarantees,,1731751082172.d9e28756f78b0555a6733ffbf6ab98bd., hostname=037c58a19ddb,37267,1731751077844, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9e28756f78b0555a6733ffbf6ab98bd, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:58:19,606 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731751082172.d9e28756f78b0555a6733ffbf6ab98bd., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9e28756f78b0555a6733ffbf6ab98bd, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:58:19,606 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731751082172.d9e28756f78b0555a6733ffbf6ab98bd., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 because the exception is null or not the one we care about 2024-11-16T09:58:19,607 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=37267 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=20 2024-11-16T09:58:19,607 DEBUG [RS_FLUSH_OPERATIONS-regionserver/037c58a19ddb:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731751082172.d9e28756f78b0555a6733ffbf6ab98bd. 2024-11-16T09:58:19,608 DEBUG [RS_FLUSH_OPERATIONS-regionserver/037c58a19ddb:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731751082172.d9e28756f78b0555a6733ffbf6ab98bd. as already flushing 2024-11-16T09:58:19,608 DEBUG [RS_FLUSH_OPERATIONS-regionserver/037c58a19ddb:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731751082172.d9e28756f78b0555a6733ffbf6ab98bd. 2024-11-16T09:58:19,608 ERROR [RS_FLUSH_OPERATIONS-regionserver/037c58a19ddb:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] handler.RSProcedureHandler(58): pid=20 java.io.IOException: Unable to complete flush {ENCODED => d9e28756f78b0555a6733ffbf6ab98bd, NAME => 'TestAcidGuarantees,,1731751082172.d9e28756f78b0555a6733ffbf6ab98bd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T09:58:19,608 DEBUG [RS_FLUSH_OPERATIONS-regionserver/037c58a19ddb:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=20 java.io.IOException: Unable to complete flush {ENCODED => d9e28756f78b0555a6733ffbf6ab98bd, NAME => 'TestAcidGuarantees,,1731751082172.d9e28756f78b0555a6733ffbf6ab98bd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T09:58:19,609 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37877 {}] master.HMaster(4177): Remote procedure failed, pid=20 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d9e28756f78b0555a6733ffbf6ab98bd, NAME => 'TestAcidGuarantees,,1731751082172.d9e28756f78b0555a6733ffbf6ab98bd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d9e28756f78b0555a6733ffbf6ab98bd, NAME => 'TestAcidGuarantees,,1731751082172.d9e28756f78b0555a6733ffbf6ab98bd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T09:58:19,712 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=55.91 KB at sequenceid=456 (bloomFilter=true), to=hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/d9e28756f78b0555a6733ffbf6ab98bd/.tmp/C/191b9d45ffa9453395b379bdb72884b4 2024-11-16T09:58:19,719 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/d9e28756f78b0555a6733ffbf6ab98bd/.tmp/A/6ba55af74e864857b6100bf2a5217c3d as hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/d9e28756f78b0555a6733ffbf6ab98bd/A/6ba55af74e864857b6100bf2a5217c3d 2024-11-16T09:58:19,724 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/d9e28756f78b0555a6733ffbf6ab98bd/A/6ba55af74e864857b6100bf2a5217c3d, entries=150, sequenceid=456, filesize=12.0 K 2024-11-16T09:58:19,726 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/d9e28756f78b0555a6733ffbf6ab98bd/.tmp/B/7c5a81154f154146ac304195f673f3de as hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/d9e28756f78b0555a6733ffbf6ab98bd/B/7c5a81154f154146ac304195f673f3de 2024-11-16T09:58:19,732 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/d9e28756f78b0555a6733ffbf6ab98bd/B/7c5a81154f154146ac304195f673f3de, entries=150, sequenceid=456, filesize=12.0 K 2024-11-16T09:58:19,733 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/d9e28756f78b0555a6733ffbf6ab98bd/.tmp/C/191b9d45ffa9453395b379bdb72884b4 as hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/d9e28756f78b0555a6733ffbf6ab98bd/C/191b9d45ffa9453395b379bdb72884b4 2024-11-16T09:58:19,739 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/d9e28756f78b0555a6733ffbf6ab98bd/C/191b9d45ffa9453395b379bdb72884b4, entries=150, sequenceid=456, filesize=12.0 K 2024-11-16T09:58:19,740 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~167.72 KB/171750, heapSize ~440.16 KB/450720, currentSize=33.54 KB/34350 for d9e28756f78b0555a6733ffbf6ab98bd in 1284ms, sequenceid=456, compaction requested=true 2024-11-16T09:58:19,740 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for d9e28756f78b0555a6733ffbf6ab98bd: 2024-11-16T09:58:19,741 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store d9e28756f78b0555a6733ffbf6ab98bd:A, priority=-2147483648, current under compaction store size is 1 2024-11-16T09:58:19,741 DEBUG [RS:0;037c58a19ddb:37267-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-16T09:58:19,741 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-16T09:58:19,741 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store d9e28756f78b0555a6733ffbf6ab98bd:B, priority=-2147483648, current under compaction store size is 2 2024-11-16T09:58:19,741 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-16T09:58:19,741 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store d9e28756f78b0555a6733ffbf6ab98bd:C, priority=-2147483648, current under compaction store size is 3 2024-11-16T09:58:19,741 DEBUG [RS:0;037c58a19ddb:37267-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-16T09:58:19,741 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-16T09:58:19,742 DEBUG [RS:0;037c58a19ddb:37267-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 55004 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-16T09:58:19,743 DEBUG [RS:0;037c58a19ddb:37267-shortCompactions-0 {}] regionserver.HStore(1541): d9e28756f78b0555a6733ffbf6ab98bd/A is initiating minor compaction (all files) 2024-11-16T09:58:19,743 INFO [RS:0;037c58a19ddb:37267-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of d9e28756f78b0555a6733ffbf6ab98bd/A in TestAcidGuarantees,,1731751082172.d9e28756f78b0555a6733ffbf6ab98bd. 2024-11-16T09:58:19,743 INFO [RS:0;037c58a19ddb:37267-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/d9e28756f78b0555a6733ffbf6ab98bd/A/2eb7658b24074f5f91f101c2bef2e6c7, hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/d9e28756f78b0555a6733ffbf6ab98bd/A/4bbd180dd1ad4a2884e953f2b35b43be, hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/d9e28756f78b0555a6733ffbf6ab98bd/A/5218094697594d79beed890deb9c28c5, hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/d9e28756f78b0555a6733ffbf6ab98bd/A/6ba55af74e864857b6100bf2a5217c3d] into tmpdir=hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/d9e28756f78b0555a6733ffbf6ab98bd/.tmp, totalSize=53.7 K 2024-11-16T09:58:19,744 DEBUG [RS:0;037c58a19ddb:37267-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 50124 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-16T09:58:19,744 DEBUG [RS:0;037c58a19ddb:37267-longCompactions-0 {}] regionserver.HStore(1541): d9e28756f78b0555a6733ffbf6ab98bd/B is initiating minor compaction (all files) 2024-11-16T09:58:19,744 INFO [RS:0;037c58a19ddb:37267-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of d9e28756f78b0555a6733ffbf6ab98bd/B in TestAcidGuarantees,,1731751082172.d9e28756f78b0555a6733ffbf6ab98bd. 2024-11-16T09:58:19,744 INFO [RS:0;037c58a19ddb:37267-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/d9e28756f78b0555a6733ffbf6ab98bd/B/a1d07b488d594516be54d1ab5a5ccf78, hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/d9e28756f78b0555a6733ffbf6ab98bd/B/b63e28d4eb564c249b118b3622ecad37, hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/d9e28756f78b0555a6733ffbf6ab98bd/B/35e2c5c8651a4291bbb7092d0b067dac, hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/d9e28756f78b0555a6733ffbf6ab98bd/B/7c5a81154f154146ac304195f673f3de] into tmpdir=hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/d9e28756f78b0555a6733ffbf6ab98bd/.tmp, totalSize=48.9 K 2024-11-16T09:58:19,744 DEBUG [RS:0;037c58a19ddb:37267-shortCompactions-0 {}] compactions.Compactor(225): Compacting 2eb7658b24074f5f91f101c2bef2e6c7, keycount=150, bloomtype=ROW, size=12.9 K, encoding=NONE, compression=NONE, seqNum=388, earliestPutTs=1731751096983 2024-11-16T09:58:19,745 DEBUG [RS:0;037c58a19ddb:37267-longCompactions-0 {}] compactions.Compactor(225): Compacting a1d07b488d594516be54d1ab5a5ccf78, keycount=150, bloomtype=ROW, size=12.9 K, encoding=NONE, compression=NONE, seqNum=388, earliestPutTs=1731751096983 2024-11-16T09:58:19,745 DEBUG [RS:0;037c58a19ddb:37267-shortCompactions-0 {}] compactions.Compactor(225): Compacting 4bbd180dd1ad4a2884e953f2b35b43be, keycount=250, bloomtype=ROW, size=16.8 K, encoding=NONE, compression=NONE, seqNum=415, earliestPutTs=1731751097016 2024-11-16T09:58:19,745 DEBUG [RS:0;037c58a19ddb:37267-longCompactions-0 {}] compactions.Compactor(225): Compacting b63e28d4eb564c249b118b3622ecad37, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=415, earliestPutTs=1731751097028 2024-11-16T09:58:19,746 DEBUG [RS:0;037c58a19ddb:37267-longCompactions-0 {}] compactions.Compactor(225): Compacting 35e2c5c8651a4291bbb7092d0b067dac, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=427, earliestPutTs=1731751097145 2024-11-16T09:58:19,746 DEBUG [RS:0;037c58a19ddb:37267-shortCompactions-0 {}] compactions.Compactor(225): Compacting 5218094697594d79beed890deb9c28c5, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=427, earliestPutTs=1731751097145 2024-11-16T09:58:19,746 DEBUG [RS:0;037c58a19ddb:37267-longCompactions-0 {}] compactions.Compactor(225): Compacting 7c5a81154f154146ac304195f673f3de, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=456, earliestPutTs=1731751097823 2024-11-16T09:58:19,747 DEBUG [RS:0;037c58a19ddb:37267-shortCompactions-0 {}] compactions.Compactor(225): Compacting 6ba55af74e864857b6100bf2a5217c3d, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=456, earliestPutTs=1731751097823 2024-11-16T09:58:19,761 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=37267 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=20 2024-11-16T09:58:19,761 DEBUG [RS_FLUSH_OPERATIONS-regionserver/037c58a19ddb:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731751082172.d9e28756f78b0555a6733ffbf6ab98bd. 2024-11-16T09:58:19,762 INFO [RS_FLUSH_OPERATIONS-regionserver/037c58a19ddb:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HRegion(2902): Flushing d9e28756f78b0555a6733ffbf6ab98bd 3/3 column families, dataSize=33.54 KB heapSize=88.64 KB 2024-11-16T09:58:19,763 INFO [RS:0;037c58a19ddb:37267-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): d9e28756f78b0555a6733ffbf6ab98bd#B#compaction#93 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-16T09:58:19,765 DEBUG [RS:0;037c58a19ddb:37267-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/d9e28756f78b0555a6733ffbf6ab98bd/.tmp/B/8f4123247f9e45bc824d823ca5119588 is 50, key is test_row_0/B:col10/1731751097823/Put/seqid=0 2024-11-16T09:58:19,770 DEBUG [RS_FLUSH_OPERATIONS-regionserver/037c58a19ddb:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/d9e28756f78b0555a6733ffbf6ab98bd/.tmp/A/4c0a53ff6af042a7ab43c7f78203daec is 50, key is test_row_0/A:col10/1731751098457/Put/seqid=0 2024-11-16T09:58:19,770 INFO [RS:0;037c58a19ddb:37267-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): d9e28756f78b0555a6733ffbf6ab98bd#A#compaction#94 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-16T09:58:19,771 DEBUG [RS:0;037c58a19ddb:37267-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/d9e28756f78b0555a6733ffbf6ab98bd/.tmp/A/6e28e90f94ae4c7cb291c53768fd3408 is 50, key is test_row_0/A:col10/1731751097823/Put/seqid=0 2024-11-16T09:58:19,796 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37591 is added to blk_1073741931_1107 (size=9857) 2024-11-16T09:58:19,797 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37591 is added to blk_1073741930_1106 (size=13357) 2024-11-16T09:58:19,801 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37591 is added to blk_1073741932_1108 (size=13357) 2024-11-16T09:58:20,005 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] regionserver.HRegion(8855): Flush requested on d9e28756f78b0555a6733ffbf6ab98bd 2024-11-16T09:58:20,005 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731751082172.d9e28756f78b0555a6733ffbf6ab98bd. as already flushing 2024-11-16T09:58:20,094 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9e28756f78b0555a6733ffbf6ab98bd, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T09:58:20,094 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] ipc.CallRunner(138): callId: 157 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39654 deadline: 1731751160093, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9e28756f78b0555a6733ffbf6ab98bd, server=037c58a19ddb,37267,1731751077844 2024-11-16T09:58:20,094 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731751082172.d9e28756f78b0555a6733ffbf6ab98bd., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 , the old value is region=TestAcidGuarantees,,1731751082172.d9e28756f78b0555a6733ffbf6ab98bd., hostname=037c58a19ddb,37267,1731751077844, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9e28756f78b0555a6733ffbf6ab98bd, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:58:20,094 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731751082172.d9e28756f78b0555a6733ffbf6ab98bd., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9e28756f78b0555a6733ffbf6ab98bd, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:58:20,094 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731751082172.d9e28756f78b0555a6733ffbf6ab98bd., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 because the exception is null or not the one we care about 2024-11-16T09:58:20,197 INFO [RS_FLUSH_OPERATIONS-regionserver/037c58a19ddb:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=464 (bloomFilter=true), to=hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/d9e28756f78b0555a6733ffbf6ab98bd/.tmp/A/4c0a53ff6af042a7ab43c7f78203daec 2024-11-16T09:58:20,205 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9e28756f78b0555a6733ffbf6ab98bd, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T09:58:20,205 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] ipc.CallRunner(138): callId: 158 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39654 deadline: 1731751160203, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9e28756f78b0555a6733ffbf6ab98bd, server=037c58a19ddb,37267,1731751077844 2024-11-16T09:58:20,205 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731751082172.d9e28756f78b0555a6733ffbf6ab98bd., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 , the old value is region=TestAcidGuarantees,,1731751082172.d9e28756f78b0555a6733ffbf6ab98bd., hostname=037c58a19ddb,37267,1731751077844, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9e28756f78b0555a6733ffbf6ab98bd, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:58:20,205 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731751082172.d9e28756f78b0555a6733ffbf6ab98bd., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9e28756f78b0555a6733ffbf6ab98bd, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:58:20,206 DEBUG [RS:0;037c58a19ddb:37267-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/d9e28756f78b0555a6733ffbf6ab98bd/.tmp/B/8f4123247f9e45bc824d823ca5119588 as hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/d9e28756f78b0555a6733ffbf6ab98bd/B/8f4123247f9e45bc824d823ca5119588 2024-11-16T09:58:20,206 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731751082172.d9e28756f78b0555a6733ffbf6ab98bd., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 because the exception is null or not the one we care about 2024-11-16T09:58:20,208 DEBUG [RS:0;037c58a19ddb:37267-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/d9e28756f78b0555a6733ffbf6ab98bd/.tmp/A/6e28e90f94ae4c7cb291c53768fd3408 as hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/d9e28756f78b0555a6733ffbf6ab98bd/A/6e28e90f94ae4c7cb291c53768fd3408 2024-11-16T09:58:20,209 DEBUG [RS_FLUSH_OPERATIONS-regionserver/037c58a19ddb:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/d9e28756f78b0555a6733ffbf6ab98bd/.tmp/B/65e44fb87deb47318c5b3f3350ad90bc is 50, key is test_row_0/B:col10/1731751098457/Put/seqid=0 2024-11-16T09:58:20,213 INFO [RS:0;037c58a19ddb:37267-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in d9e28756f78b0555a6733ffbf6ab98bd/B of d9e28756f78b0555a6733ffbf6ab98bd into 8f4123247f9e45bc824d823ca5119588(size=13.0 K), total size for store is 13.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-16T09:58:20,213 DEBUG [RS:0;037c58a19ddb:37267-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for d9e28756f78b0555a6733ffbf6ab98bd: 2024-11-16T09:58:20,213 INFO [RS:0;037c58a19ddb:37267-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731751082172.d9e28756f78b0555a6733ffbf6ab98bd., storeName=d9e28756f78b0555a6733ffbf6ab98bd/B, priority=12, startTime=1731751099741; duration=0sec 2024-11-16T09:58:20,214 DEBUG [RS:0;037c58a19ddb:37267-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-16T09:58:20,214 DEBUG [RS:0;037c58a19ddb:37267-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: d9e28756f78b0555a6733ffbf6ab98bd:B 2024-11-16T09:58:20,214 DEBUG [RS:0;037c58a19ddb:37267-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-16T09:58:20,216 DEBUG [RS:0;037c58a19ddb:37267-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 50124 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-16T09:58:20,216 DEBUG [RS:0;037c58a19ddb:37267-longCompactions-0 {}] regionserver.HStore(1541): d9e28756f78b0555a6733ffbf6ab98bd/C is initiating minor compaction (all files) 2024-11-16T09:58:20,216 INFO [RS:0;037c58a19ddb:37267-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of d9e28756f78b0555a6733ffbf6ab98bd/C in TestAcidGuarantees,,1731751082172.d9e28756f78b0555a6733ffbf6ab98bd. 2024-11-16T09:58:20,216 INFO [RS:0;037c58a19ddb:37267-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/d9e28756f78b0555a6733ffbf6ab98bd/C/bafc286da4bb4b178dc9fe040d3351f4, hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/d9e28756f78b0555a6733ffbf6ab98bd/C/6abf151a60c5402f91aa7af92dc02406, hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/d9e28756f78b0555a6733ffbf6ab98bd/C/0f9fc7e9d0654382b9999d87e440edee, hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/d9e28756f78b0555a6733ffbf6ab98bd/C/191b9d45ffa9453395b379bdb72884b4] into tmpdir=hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/d9e28756f78b0555a6733ffbf6ab98bd/.tmp, totalSize=48.9 K 2024-11-16T09:58:20,216 INFO [RS:0;037c58a19ddb:37267-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in d9e28756f78b0555a6733ffbf6ab98bd/A of d9e28756f78b0555a6733ffbf6ab98bd into 6e28e90f94ae4c7cb291c53768fd3408(size=13.0 K), total size for store is 13.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-16T09:58:20,216 DEBUG [RS:0;037c58a19ddb:37267-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for d9e28756f78b0555a6733ffbf6ab98bd: 2024-11-16T09:58:20,216 INFO [RS:0;037c58a19ddb:37267-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731751082172.d9e28756f78b0555a6733ffbf6ab98bd., storeName=d9e28756f78b0555a6733ffbf6ab98bd/A, priority=12, startTime=1731751099741; duration=0sec 2024-11-16T09:58:20,216 DEBUG [RS:0;037c58a19ddb:37267-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-16T09:58:20,217 DEBUG [RS:0;037c58a19ddb:37267-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: d9e28756f78b0555a6733ffbf6ab98bd:A 2024-11-16T09:58:20,217 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37591 is added to blk_1073741933_1109 (size=9857) 2024-11-16T09:58:20,217 DEBUG [RS:0;037c58a19ddb:37267-longCompactions-0 {}] compactions.Compactor(225): Compacting bafc286da4bb4b178dc9fe040d3351f4, keycount=150, bloomtype=ROW, size=12.9 K, encoding=NONE, compression=NONE, seqNum=388, earliestPutTs=1731751096983 2024-11-16T09:58:20,218 DEBUG [RS:0;037c58a19ddb:37267-longCompactions-0 {}] compactions.Compactor(225): Compacting 6abf151a60c5402f91aa7af92dc02406, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=415, earliestPutTs=1731751097028 2024-11-16T09:58:20,218 INFO [RS_FLUSH_OPERATIONS-regionserver/037c58a19ddb:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=464 (bloomFilter=true), to=hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/d9e28756f78b0555a6733ffbf6ab98bd/.tmp/B/65e44fb87deb47318c5b3f3350ad90bc 2024-11-16T09:58:20,218 DEBUG [RS:0;037c58a19ddb:37267-longCompactions-0 {}] compactions.Compactor(225): Compacting 0f9fc7e9d0654382b9999d87e440edee, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=427, earliestPutTs=1731751097145 2024-11-16T09:58:20,219 DEBUG [RS:0;037c58a19ddb:37267-longCompactions-0 {}] compactions.Compactor(225): Compacting 191b9d45ffa9453395b379bdb72884b4, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=456, earliestPutTs=1731751097823 2024-11-16T09:58:20,234 DEBUG [RS_FLUSH_OPERATIONS-regionserver/037c58a19ddb:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/d9e28756f78b0555a6733ffbf6ab98bd/.tmp/C/ec2fc03b717144878441c4f35d7afb67 is 50, key is test_row_0/C:col10/1731751098457/Put/seqid=0 2024-11-16T09:58:20,235 INFO [RS:0;037c58a19ddb:37267-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): d9e28756f78b0555a6733ffbf6ab98bd#C#compaction#98 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-16T09:58:20,236 DEBUG [RS:0;037c58a19ddb:37267-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/d9e28756f78b0555a6733ffbf6ab98bd/.tmp/C/d71cf2b421514dbf9ac92fd09d0b6add is 50, key is test_row_0/C:col10/1731751097823/Put/seqid=0 2024-11-16T09:58:20,248 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37591 is added to blk_1073741935_1111 (size=13357) 2024-11-16T09:58:20,250 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37591 is added to blk_1073741934_1110 (size=9857) 2024-11-16T09:58:20,413 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9e28756f78b0555a6733ffbf6ab98bd, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T09:58:20,413 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] ipc.CallRunner(138): callId: 159 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39654 deadline: 1731751160413, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9e28756f78b0555a6733ffbf6ab98bd, server=037c58a19ddb,37267,1731751077844 2024-11-16T09:58:20,414 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731751082172.d9e28756f78b0555a6733ffbf6ab98bd., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 , the old value is region=TestAcidGuarantees,,1731751082172.d9e28756f78b0555a6733ffbf6ab98bd., hostname=037c58a19ddb,37267,1731751077844, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9e28756f78b0555a6733ffbf6ab98bd, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:58:20,414 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731751082172.d9e28756f78b0555a6733ffbf6ab98bd., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9e28756f78b0555a6733ffbf6ab98bd, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:58:20,414 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731751082172.d9e28756f78b0555a6733ffbf6ab98bd., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 because the exception is null or not the one we care about 2024-11-16T09:58:20,523 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37877 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=19 2024-11-16T09:58:20,617 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9e28756f78b0555a6733ffbf6ab98bd, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T09:58:20,617 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] ipc.CallRunner(138): callId: 134 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39696 deadline: 1731751160612, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9e28756f78b0555a6733ffbf6ab98bd, server=037c58a19ddb,37267,1731751077844 2024-11-16T09:58:20,617 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9e28756f78b0555a6733ffbf6ab98bd, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T09:58:20,618 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] ipc.CallRunner(138): callId: 129 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39638 deadline: 1731751160613, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9e28756f78b0555a6733ffbf6ab98bd, server=037c58a19ddb,37267,1731751077844 2024-11-16T09:58:20,618 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9e28756f78b0555a6733ffbf6ab98bd, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T09:58:20,618 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] ipc.CallRunner(138): callId: 133 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39630 deadline: 1731751160613, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9e28756f78b0555a6733ffbf6ab98bd, server=037c58a19ddb,37267,1731751077844 2024-11-16T09:58:20,621 WARN [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRpcRetryingCaller(177): Call to 037c58a19ddb,37267,1731751077844 for 'test_row_0' in d9e28756f78b0555a6733ffbf6ab98bd of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2156 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9e28756f78b0555a6733ffbf6ab98bd, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor44.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9e28756f78b0555a6733ffbf6ab98bd, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-16T09:58:20,622 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731751082172.d9e28756f78b0555a6733ffbf6ab98bd., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 , the old value is region=TestAcidGuarantees,,1731751082172.d9e28756f78b0555a6733ffbf6ab98bd., hostname=037c58a19ddb,37267,1731751077844, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9e28756f78b0555a6733ffbf6ab98bd, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:58:20,622 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731751082172.d9e28756f78b0555a6733ffbf6ab98bd., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9e28756f78b0555a6733ffbf6ab98bd, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:58:20,623 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731751082172.d9e28756f78b0555a6733ffbf6ab98bd., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 because the exception is null or not the one we care about 2024-11-16T09:58:20,621 WARN [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRpcRetryingCaller(177): Call to 037c58a19ddb,37267,1731751077844 for 'test_row_0' in d9e28756f78b0555a6733ffbf6ab98bd of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2153 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9e28756f78b0555a6733ffbf6ab98bd, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor44.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9e28756f78b0555a6733ffbf6ab98bd, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-16T09:58:20,623 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731751082172.d9e28756f78b0555a6733ffbf6ab98bd., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 , the old value is region=TestAcidGuarantees,,1731751082172.d9e28756f78b0555a6733ffbf6ab98bd., hostname=037c58a19ddb,37267,1731751077844, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9e28756f78b0555a6733ffbf6ab98bd, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:58:20,623 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731751082172.d9e28756f78b0555a6733ffbf6ab98bd., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9e28756f78b0555a6733ffbf6ab98bd, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:58:20,623 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731751082172.d9e28756f78b0555a6733ffbf6ab98bd., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 because the exception is null or not the one we care about 2024-11-16T09:58:20,623 WARN [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRpcRetryingCaller(177): Call to 037c58a19ddb,37267,1731751077844 for 'test_row_2' in d9e28756f78b0555a6733ffbf6ab98bd of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2159 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9e28756f78b0555a6733ffbf6ab98bd, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor44.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9e28756f78b0555a6733ffbf6ab98bd, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-16T09:58:20,624 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731751082172.d9e28756f78b0555a6733ffbf6ab98bd., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 , the old value is region=TestAcidGuarantees,,1731751082172.d9e28756f78b0555a6733ffbf6ab98bd., hostname=037c58a19ddb,37267,1731751077844, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9e28756f78b0555a6733ffbf6ab98bd, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:58:20,624 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731751082172.d9e28756f78b0555a6733ffbf6ab98bd., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9e28756f78b0555a6733ffbf6ab98bd, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:58:20,624 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731751082172.d9e28756f78b0555a6733ffbf6ab98bd., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 because the exception is null or not the one we care about 2024-11-16T09:58:20,628 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9e28756f78b0555a6733ffbf6ab98bd, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T09:58:20,628 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] ipc.CallRunner(138): callId: 136 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39620 deadline: 1731751160624, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9e28756f78b0555a6733ffbf6ab98bd, server=037c58a19ddb,37267,1731751077844 2024-11-16T09:58:20,629 WARN [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRpcRetryingCaller(177): Call to 037c58a19ddb,37267,1731751077844 for 'test_row_1' in d9e28756f78b0555a6733ffbf6ab98bd of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2170 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9e28756f78b0555a6733ffbf6ab98bd, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor44.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9e28756f78b0555a6733ffbf6ab98bd, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-16T09:58:20,630 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731751082172.d9e28756f78b0555a6733ffbf6ab98bd., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 , the old value is region=TestAcidGuarantees,,1731751082172.d9e28756f78b0555a6733ffbf6ab98bd., hostname=037c58a19ddb,37267,1731751077844, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9e28756f78b0555a6733ffbf6ab98bd, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:58:20,630 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731751082172.d9e28756f78b0555a6733ffbf6ab98bd., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9e28756f78b0555a6733ffbf6ab98bd, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:58:20,630 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731751082172.d9e28756f78b0555a6733ffbf6ab98bd., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 because the exception is null or not the one we care about 2024-11-16T09:58:20,652 INFO [RS_FLUSH_OPERATIONS-regionserver/037c58a19ddb:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=464 (bloomFilter=true), to=hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/d9e28756f78b0555a6733ffbf6ab98bd/.tmp/C/ec2fc03b717144878441c4f35d7afb67 2024-11-16T09:58:20,662 DEBUG [RS:0;037c58a19ddb:37267-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/d9e28756f78b0555a6733ffbf6ab98bd/.tmp/C/d71cf2b421514dbf9ac92fd09d0b6add as hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/d9e28756f78b0555a6733ffbf6ab98bd/C/d71cf2b421514dbf9ac92fd09d0b6add 2024-11-16T09:58:20,663 DEBUG [RS_FLUSH_OPERATIONS-regionserver/037c58a19ddb:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/d9e28756f78b0555a6733ffbf6ab98bd/.tmp/A/4c0a53ff6af042a7ab43c7f78203daec as hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/d9e28756f78b0555a6733ffbf6ab98bd/A/4c0a53ff6af042a7ab43c7f78203daec 2024-11-16T09:58:20,674 INFO [RS_FLUSH_OPERATIONS-regionserver/037c58a19ddb:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/d9e28756f78b0555a6733ffbf6ab98bd/A/4c0a53ff6af042a7ab43c7f78203daec, entries=100, sequenceid=464, filesize=9.6 K 2024-11-16T09:58:20,674 INFO [RS:0;037c58a19ddb:37267-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in d9e28756f78b0555a6733ffbf6ab98bd/C of d9e28756f78b0555a6733ffbf6ab98bd into d71cf2b421514dbf9ac92fd09d0b6add(size=13.0 K), total size for store is 13.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-16T09:58:20,674 DEBUG [RS:0;037c58a19ddb:37267-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for d9e28756f78b0555a6733ffbf6ab98bd: 2024-11-16T09:58:20,674 INFO [RS:0;037c58a19ddb:37267-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731751082172.d9e28756f78b0555a6733ffbf6ab98bd., storeName=d9e28756f78b0555a6733ffbf6ab98bd/C, priority=12, startTime=1731751099741; duration=0sec 2024-11-16T09:58:20,674 DEBUG [RS:0;037c58a19ddb:37267-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-16T09:58:20,675 DEBUG [RS:0;037c58a19ddb:37267-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: d9e28756f78b0555a6733ffbf6ab98bd:C 2024-11-16T09:58:20,675 DEBUG [RS_FLUSH_OPERATIONS-regionserver/037c58a19ddb:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/d9e28756f78b0555a6733ffbf6ab98bd/.tmp/B/65e44fb87deb47318c5b3f3350ad90bc as hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/d9e28756f78b0555a6733ffbf6ab98bd/B/65e44fb87deb47318c5b3f3350ad90bc 2024-11-16T09:58:20,681 INFO [RS_FLUSH_OPERATIONS-regionserver/037c58a19ddb:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/d9e28756f78b0555a6733ffbf6ab98bd/B/65e44fb87deb47318c5b3f3350ad90bc, entries=100, sequenceid=464, filesize=9.6 K 2024-11-16T09:58:20,682 DEBUG [RS_FLUSH_OPERATIONS-regionserver/037c58a19ddb:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/d9e28756f78b0555a6733ffbf6ab98bd/.tmp/C/ec2fc03b717144878441c4f35d7afb67 as hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/d9e28756f78b0555a6733ffbf6ab98bd/C/ec2fc03b717144878441c4f35d7afb67 2024-11-16T09:58:20,686 INFO [RS_FLUSH_OPERATIONS-regionserver/037c58a19ddb:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/d9e28756f78b0555a6733ffbf6ab98bd/C/ec2fc03b717144878441c4f35d7afb67, entries=100, sequenceid=464, filesize=9.6 K 2024-11-16T09:58:20,688 INFO [RS_FLUSH_OPERATIONS-regionserver/037c58a19ddb:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HRegion(3140): Finished flush of dataSize ~33.54 KB/34350, heapSize ~88.59 KB/90720, currentSize=167.72 KB/171750 for d9e28756f78b0555a6733ffbf6ab98bd in 927ms, sequenceid=464, compaction requested=false 2024-11-16T09:58:20,688 DEBUG [RS_FLUSH_OPERATIONS-regionserver/037c58a19ddb:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HRegion(2603): Flush status journal for d9e28756f78b0555a6733ffbf6ab98bd: 2024-11-16T09:58:20,688 DEBUG [RS_FLUSH_OPERATIONS-regionserver/037c58a19ddb:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731751082172.d9e28756f78b0555a6733ffbf6ab98bd. 2024-11-16T09:58:20,688 DEBUG [RS_FLUSH_OPERATIONS-regionserver/037c58a19ddb:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=20 2024-11-16T09:58:20,688 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37877 {}] master.HMaster(4169): Remote procedure done, pid=20 2024-11-16T09:58:20,692 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=20, resume processing ppid=19 2024-11-16T09:58:20,692 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=20, ppid=19, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.3130 sec 2024-11-16T09:58:20,696 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=19, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=19, table=TestAcidGuarantees in 2.3200 sec 2024-11-16T09:58:20,725 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] regionserver.HRegion(8855): Flush requested on d9e28756f78b0555a6733ffbf6ab98bd 2024-11-16T09:58:20,725 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-16T09:58:20,725 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing d9e28756f78b0555a6733ffbf6ab98bd 3/3 column families, dataSize=174.43 KB heapSize=457.78 KB 2024-11-16T09:58:20,731 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/d9e28756f78b0555a6733ffbf6ab98bd/.tmp/A/92a11abe45c74a93a787b68e82adc154 is 50, key is test_row_0/A:col10/1731751100091/Put/seqid=0 2024-11-16T09:58:20,737 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37591 is added to blk_1073741936_1112 (size=12301) 2024-11-16T09:58:20,740 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9e28756f78b0555a6733ffbf6ab98bd, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T09:58:20,741 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] ipc.CallRunner(138): callId: 165 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39654 deadline: 1731751160738, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9e28756f78b0555a6733ffbf6ab98bd, server=037c58a19ddb,37267,1731751077844 2024-11-16T09:58:20,741 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731751082172.d9e28756f78b0555a6733ffbf6ab98bd., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 , the old value is region=TestAcidGuarantees,,1731751082172.d9e28756f78b0555a6733ffbf6ab98bd., hostname=037c58a19ddb,37267,1731751077844, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9e28756f78b0555a6733ffbf6ab98bd, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:58:20,741 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731751082172.d9e28756f78b0555a6733ffbf6ab98bd., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9e28756f78b0555a6733ffbf6ab98bd, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:58:20,741 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731751082172.d9e28756f78b0555a6733ffbf6ab98bd., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 because the exception is null or not the one we care about 2024-11-16T09:58:20,856 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9e28756f78b0555a6733ffbf6ab98bd, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T09:58:20,856 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] ipc.CallRunner(138): callId: 166 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39654 deadline: 1731751160853, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9e28756f78b0555a6733ffbf6ab98bd, server=037c58a19ddb,37267,1731751077844 2024-11-16T09:58:20,857 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731751082172.d9e28756f78b0555a6733ffbf6ab98bd., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 , the old value is region=TestAcidGuarantees,,1731751082172.d9e28756f78b0555a6733ffbf6ab98bd., hostname=037c58a19ddb,37267,1731751077844, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9e28756f78b0555a6733ffbf6ab98bd, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:58:20,857 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731751082172.d9e28756f78b0555a6733ffbf6ab98bd., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9e28756f78b0555a6733ffbf6ab98bd, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:58:20,857 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731751082172.d9e28756f78b0555a6733ffbf6ab98bd., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 because the exception is null or not the one we care about 2024-11-16T09:58:21,063 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9e28756f78b0555a6733ffbf6ab98bd, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T09:58:21,063 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] ipc.CallRunner(138): callId: 167 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39654 deadline: 1731751161063, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9e28756f78b0555a6733ffbf6ab98bd, server=037c58a19ddb,37267,1731751077844 2024-11-16T09:58:21,064 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731751082172.d9e28756f78b0555a6733ffbf6ab98bd., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 , the old value is region=TestAcidGuarantees,,1731751082172.d9e28756f78b0555a6733ffbf6ab98bd., hostname=037c58a19ddb,37267,1731751077844, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9e28756f78b0555a6733ffbf6ab98bd, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:58:21,064 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731751082172.d9e28756f78b0555a6733ffbf6ab98bd., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9e28756f78b0555a6733ffbf6ab98bd, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:58:21,064 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731751082172.d9e28756f78b0555a6733ffbf6ab98bd., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 because the exception is null or not the one we care about 2024-11-16T09:58:21,138 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=58.14 KB at sequenceid=496 (bloomFilter=true), to=hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/d9e28756f78b0555a6733ffbf6ab98bd/.tmp/A/92a11abe45c74a93a787b68e82adc154 2024-11-16T09:58:21,147 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/d9e28756f78b0555a6733ffbf6ab98bd/.tmp/B/d738eb57280e4a3d87fdad85e5a6abb4 is 50, key is test_row_0/B:col10/1731751100091/Put/seqid=0 2024-11-16T09:58:21,152 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37591 is added to blk_1073741937_1113 (size=12301) 2024-11-16T09:58:21,374 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9e28756f78b0555a6733ffbf6ab98bd, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T09:58:21,374 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] ipc.CallRunner(138): callId: 168 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39654 deadline: 1731751161372, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9e28756f78b0555a6733ffbf6ab98bd, server=037c58a19ddb,37267,1731751077844 2024-11-16T09:58:21,374 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731751082172.d9e28756f78b0555a6733ffbf6ab98bd., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 , the old value is region=TestAcidGuarantees,,1731751082172.d9e28756f78b0555a6733ffbf6ab98bd., hostname=037c58a19ddb,37267,1731751077844, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9e28756f78b0555a6733ffbf6ab98bd, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:58:21,374 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731751082172.d9e28756f78b0555a6733ffbf6ab98bd., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9e28756f78b0555a6733ffbf6ab98bd, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:58:21,374 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731751082172.d9e28756f78b0555a6733ffbf6ab98bd., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 because the exception is null or not the one we care about 2024-11-16T09:58:21,553 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=58.14 KB at sequenceid=496 (bloomFilter=true), to=hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/d9e28756f78b0555a6733ffbf6ab98bd/.tmp/B/d738eb57280e4a3d87fdad85e5a6abb4 2024-11-16T09:58:21,565 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/d9e28756f78b0555a6733ffbf6ab98bd/.tmp/C/d9504800b71f4da08755d6f66cbed8ab is 50, key is test_row_0/C:col10/1731751100091/Put/seqid=0 2024-11-16T09:58:21,581 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37591 is added to blk_1073741938_1114 (size=12301) 2024-11-16T09:58:21,582 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=58.14 KB at sequenceid=496 (bloomFilter=true), to=hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/d9e28756f78b0555a6733ffbf6ab98bd/.tmp/C/d9504800b71f4da08755d6f66cbed8ab 2024-11-16T09:58:21,591 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/d9e28756f78b0555a6733ffbf6ab98bd/.tmp/A/92a11abe45c74a93a787b68e82adc154 as hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/d9e28756f78b0555a6733ffbf6ab98bd/A/92a11abe45c74a93a787b68e82adc154 2024-11-16T09:58:21,605 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/d9e28756f78b0555a6733ffbf6ab98bd/A/92a11abe45c74a93a787b68e82adc154, entries=150, sequenceid=496, filesize=12.0 K 2024-11-16T09:58:21,606 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/d9e28756f78b0555a6733ffbf6ab98bd/.tmp/B/d738eb57280e4a3d87fdad85e5a6abb4 as hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/d9e28756f78b0555a6733ffbf6ab98bd/B/d738eb57280e4a3d87fdad85e5a6abb4 2024-11-16T09:58:21,613 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/d9e28756f78b0555a6733ffbf6ab98bd/B/d738eb57280e4a3d87fdad85e5a6abb4, entries=150, sequenceid=496, filesize=12.0 K 2024-11-16T09:58:21,614 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/d9e28756f78b0555a6733ffbf6ab98bd/.tmp/C/d9504800b71f4da08755d6f66cbed8ab as hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/d9e28756f78b0555a6733ffbf6ab98bd/C/d9504800b71f4da08755d6f66cbed8ab 2024-11-16T09:58:21,620 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/d9e28756f78b0555a6733ffbf6ab98bd/C/d9504800b71f4da08755d6f66cbed8ab, entries=150, sequenceid=496, filesize=12.0 K 2024-11-16T09:58:21,622 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~174.43 KB/178620, heapSize ~457.73 KB/468720, currentSize=26.84 KB/27480 for d9e28756f78b0555a6733ffbf6ab98bd in 897ms, sequenceid=496, compaction requested=true 2024-11-16T09:58:21,622 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for d9e28756f78b0555a6733ffbf6ab98bd: 2024-11-16T09:58:21,624 DEBUG [RS:0;037c58a19ddb:37267-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-16T09:58:21,624 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store d9e28756f78b0555a6733ffbf6ab98bd:A, priority=-2147483648, current under compaction store size is 1 2024-11-16T09:58:21,624 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-16T09:58:21,625 DEBUG [RS:0;037c58a19ddb:37267-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-16T09:58:21,625 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store d9e28756f78b0555a6733ffbf6ab98bd:B, priority=-2147483648, current under compaction store size is 2 2024-11-16T09:58:21,625 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-16T09:58:21,625 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store d9e28756f78b0555a6733ffbf6ab98bd:C, priority=-2147483648, current under compaction store size is 3 2024-11-16T09:58:21,626 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-16T09:58:21,626 DEBUG [RS:0;037c58a19ddb:37267-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 35515 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-16T09:58:21,626 DEBUG [RS:0;037c58a19ddb:37267-shortCompactions-0 {}] regionserver.HStore(1541): d9e28756f78b0555a6733ffbf6ab98bd/A is initiating minor compaction (all files) 2024-11-16T09:58:21,626 INFO [RS:0;037c58a19ddb:37267-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of d9e28756f78b0555a6733ffbf6ab98bd/A in TestAcidGuarantees,,1731751082172.d9e28756f78b0555a6733ffbf6ab98bd. 2024-11-16T09:58:21,626 INFO [RS:0;037c58a19ddb:37267-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/d9e28756f78b0555a6733ffbf6ab98bd/A/6e28e90f94ae4c7cb291c53768fd3408, hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/d9e28756f78b0555a6733ffbf6ab98bd/A/4c0a53ff6af042a7ab43c7f78203daec, hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/d9e28756f78b0555a6733ffbf6ab98bd/A/92a11abe45c74a93a787b68e82adc154] into tmpdir=hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/d9e28756f78b0555a6733ffbf6ab98bd/.tmp, totalSize=34.7 K 2024-11-16T09:58:21,626 DEBUG [RS:0;037c58a19ddb:37267-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 35515 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-16T09:58:21,626 DEBUG [RS:0;037c58a19ddb:37267-longCompactions-0 {}] regionserver.HStore(1541): d9e28756f78b0555a6733ffbf6ab98bd/B is initiating minor compaction (all files) 2024-11-16T09:58:21,627 INFO [RS:0;037c58a19ddb:37267-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of d9e28756f78b0555a6733ffbf6ab98bd/B in TestAcidGuarantees,,1731751082172.d9e28756f78b0555a6733ffbf6ab98bd. 2024-11-16T09:58:21,627 DEBUG [RS:0;037c58a19ddb:37267-shortCompactions-0 {}] compactions.Compactor(225): Compacting 6e28e90f94ae4c7cb291c53768fd3408, keycount=150, bloomtype=ROW, size=13.0 K, encoding=NONE, compression=NONE, seqNum=456, earliestPutTs=1731751097823 2024-11-16T09:58:21,627 INFO [RS:0;037c58a19ddb:37267-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/d9e28756f78b0555a6733ffbf6ab98bd/B/8f4123247f9e45bc824d823ca5119588, hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/d9e28756f78b0555a6733ffbf6ab98bd/B/65e44fb87deb47318c5b3f3350ad90bc, hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/d9e28756f78b0555a6733ffbf6ab98bd/B/d738eb57280e4a3d87fdad85e5a6abb4] into tmpdir=hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/d9e28756f78b0555a6733ffbf6ab98bd/.tmp, totalSize=34.7 K 2024-11-16T09:58:21,627 DEBUG [RS:0;037c58a19ddb:37267-shortCompactions-0 {}] compactions.Compactor(225): Compacting 4c0a53ff6af042a7ab43c7f78203daec, keycount=100, bloomtype=ROW, size=9.6 K, encoding=NONE, compression=NONE, seqNum=464, earliestPutTs=1731751098457 2024-11-16T09:58:21,627 DEBUG [RS:0;037c58a19ddb:37267-longCompactions-0 {}] compactions.Compactor(225): Compacting 8f4123247f9e45bc824d823ca5119588, keycount=150, bloomtype=ROW, size=13.0 K, encoding=NONE, compression=NONE, seqNum=456, earliestPutTs=1731751097823 2024-11-16T09:58:21,628 DEBUG [RS:0;037c58a19ddb:37267-shortCompactions-0 {}] compactions.Compactor(225): Compacting 92a11abe45c74a93a787b68e82adc154, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=496, earliestPutTs=1731751100079 2024-11-16T09:58:21,628 DEBUG [RS:0;037c58a19ddb:37267-longCompactions-0 {}] compactions.Compactor(225): Compacting 65e44fb87deb47318c5b3f3350ad90bc, keycount=100, bloomtype=ROW, size=9.6 K, encoding=NONE, compression=NONE, seqNum=464, earliestPutTs=1731751098457 2024-11-16T09:58:21,628 DEBUG [RS:0;037c58a19ddb:37267-longCompactions-0 {}] compactions.Compactor(225): Compacting d738eb57280e4a3d87fdad85e5a6abb4, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=496, earliestPutTs=1731751100079 2024-11-16T09:58:21,638 INFO [RS:0;037c58a19ddb:37267-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): d9e28756f78b0555a6733ffbf6ab98bd#B#compaction#102 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-16T09:58:21,639 DEBUG [RS:0;037c58a19ddb:37267-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/d9e28756f78b0555a6733ffbf6ab98bd/.tmp/B/bc86639a0a354f0aaec9fc1c395896b8 is 50, key is test_row_0/B:col10/1731751100091/Put/seqid=0 2024-11-16T09:58:21,646 INFO [RS:0;037c58a19ddb:37267-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): d9e28756f78b0555a6733ffbf6ab98bd#A#compaction#103 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-16T09:58:21,647 DEBUG [RS:0;037c58a19ddb:37267-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/d9e28756f78b0555a6733ffbf6ab98bd/.tmp/A/f84ef8c17537456cb6d451fa35e32d14 is 50, key is test_row_0/A:col10/1731751100091/Put/seqid=0 2024-11-16T09:58:21,664 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37591 is added to blk_1073741939_1115 (size=13459) 2024-11-16T09:58:21,675 DEBUG [RS:0;037c58a19ddb:37267-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/d9e28756f78b0555a6733ffbf6ab98bd/.tmp/B/bc86639a0a354f0aaec9fc1c395896b8 as hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/d9e28756f78b0555a6733ffbf6ab98bd/B/bc86639a0a354f0aaec9fc1c395896b8 2024-11-16T09:58:21,677 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37591 is added to blk_1073741940_1116 (size=13459) 2024-11-16T09:58:21,683 INFO [RS:0;037c58a19ddb:37267-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in d9e28756f78b0555a6733ffbf6ab98bd/B of d9e28756f78b0555a6733ffbf6ab98bd into bc86639a0a354f0aaec9fc1c395896b8(size=13.1 K), total size for store is 13.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-16T09:58:21,683 DEBUG [RS:0;037c58a19ddb:37267-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for d9e28756f78b0555a6733ffbf6ab98bd: 2024-11-16T09:58:21,683 INFO [RS:0;037c58a19ddb:37267-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731751082172.d9e28756f78b0555a6733ffbf6ab98bd., storeName=d9e28756f78b0555a6733ffbf6ab98bd/B, priority=13, startTime=1731751101624; duration=0sec 2024-11-16T09:58:21,683 DEBUG [RS:0;037c58a19ddb:37267-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-16T09:58:21,683 DEBUG [RS:0;037c58a19ddb:37267-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: d9e28756f78b0555a6733ffbf6ab98bd:B 2024-11-16T09:58:21,683 DEBUG [RS:0;037c58a19ddb:37267-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-16T09:58:21,685 DEBUG [RS:0;037c58a19ddb:37267-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 35515 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-16T09:58:21,685 DEBUG [RS:0;037c58a19ddb:37267-longCompactions-0 {}] regionserver.HStore(1541): d9e28756f78b0555a6733ffbf6ab98bd/C is initiating minor compaction (all files) 2024-11-16T09:58:21,685 DEBUG [RS:0;037c58a19ddb:37267-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/d9e28756f78b0555a6733ffbf6ab98bd/.tmp/A/f84ef8c17537456cb6d451fa35e32d14 as hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/d9e28756f78b0555a6733ffbf6ab98bd/A/f84ef8c17537456cb6d451fa35e32d14 2024-11-16T09:58:21,685 INFO [RS:0;037c58a19ddb:37267-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of d9e28756f78b0555a6733ffbf6ab98bd/C in TestAcidGuarantees,,1731751082172.d9e28756f78b0555a6733ffbf6ab98bd. 2024-11-16T09:58:21,685 INFO [RS:0;037c58a19ddb:37267-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/d9e28756f78b0555a6733ffbf6ab98bd/C/d71cf2b421514dbf9ac92fd09d0b6add, hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/d9e28756f78b0555a6733ffbf6ab98bd/C/ec2fc03b717144878441c4f35d7afb67, hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/d9e28756f78b0555a6733ffbf6ab98bd/C/d9504800b71f4da08755d6f66cbed8ab] into tmpdir=hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/d9e28756f78b0555a6733ffbf6ab98bd/.tmp, totalSize=34.7 K 2024-11-16T09:58:21,686 DEBUG [RS:0;037c58a19ddb:37267-longCompactions-0 {}] compactions.Compactor(225): Compacting d71cf2b421514dbf9ac92fd09d0b6add, keycount=150, bloomtype=ROW, size=13.0 K, encoding=NONE, compression=NONE, seqNum=456, earliestPutTs=1731751097823 2024-11-16T09:58:21,686 DEBUG [RS:0;037c58a19ddb:37267-longCompactions-0 {}] compactions.Compactor(225): Compacting ec2fc03b717144878441c4f35d7afb67, keycount=100, bloomtype=ROW, size=9.6 K, encoding=NONE, compression=NONE, seqNum=464, earliestPutTs=1731751098457 2024-11-16T09:58:21,687 DEBUG [RS:0;037c58a19ddb:37267-longCompactions-0 {}] compactions.Compactor(225): Compacting d9504800b71f4da08755d6f66cbed8ab, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=496, earliestPutTs=1731751100079 2024-11-16T09:58:21,692 INFO [RS:0;037c58a19ddb:37267-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in d9e28756f78b0555a6733ffbf6ab98bd/A of d9e28756f78b0555a6733ffbf6ab98bd into f84ef8c17537456cb6d451fa35e32d14(size=13.1 K), total size for store is 13.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-16T09:58:21,692 DEBUG [RS:0;037c58a19ddb:37267-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for d9e28756f78b0555a6733ffbf6ab98bd: 2024-11-16T09:58:21,692 INFO [RS:0;037c58a19ddb:37267-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731751082172.d9e28756f78b0555a6733ffbf6ab98bd., storeName=d9e28756f78b0555a6733ffbf6ab98bd/A, priority=13, startTime=1731751101624; duration=0sec 2024-11-16T09:58:21,692 DEBUG [RS:0;037c58a19ddb:37267-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-16T09:58:21,693 DEBUG [RS:0;037c58a19ddb:37267-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: d9e28756f78b0555a6733ffbf6ab98bd:A 2024-11-16T09:58:21,698 INFO [RS:0;037c58a19ddb:37267-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): d9e28756f78b0555a6733ffbf6ab98bd#C#compaction#104 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-16T09:58:21,700 DEBUG [RS:0;037c58a19ddb:37267-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/d9e28756f78b0555a6733ffbf6ab98bd/.tmp/C/95c0d63049154ac794ce470e2eb2ed66 is 50, key is test_row_0/C:col10/1731751100091/Put/seqid=0 2024-11-16T09:58:21,708 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37591 is added to blk_1073741941_1117 (size=13459) 2024-11-16T09:58:21,891 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] regionserver.HRegion(8855): Flush requested on d9e28756f78b0555a6733ffbf6ab98bd 2024-11-16T09:58:21,891 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-16T09:58:21,891 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing d9e28756f78b0555a6733ffbf6ab98bd 3/3 column families, dataSize=53.67 KB heapSize=141.38 KB 2024-11-16T09:58:21,899 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/d9e28756f78b0555a6733ffbf6ab98bd/.tmp/A/2d6792fd6ed14e2abcfec73afc105145 is 50, key is test_row_0/A:col10/1731751101886/Put/seqid=0 2024-11-16T09:58:21,906 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37591 is added to blk_1073741942_1118 (size=9857) 2024-11-16T09:58:21,954 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9e28756f78b0555a6733ffbf6ab98bd, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T09:58:21,954 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] ipc.CallRunner(138): callId: 195 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39654 deadline: 1731751161949, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9e28756f78b0555a6733ffbf6ab98bd, server=037c58a19ddb,37267,1731751077844 2024-11-16T09:58:21,955 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731751082172.d9e28756f78b0555a6733ffbf6ab98bd., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 , the old value is region=TestAcidGuarantees,,1731751082172.d9e28756f78b0555a6733ffbf6ab98bd., hostname=037c58a19ddb,37267,1731751077844, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9e28756f78b0555a6733ffbf6ab98bd, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:58:21,955 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731751082172.d9e28756f78b0555a6733ffbf6ab98bd., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9e28756f78b0555a6733ffbf6ab98bd, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:58:21,955 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731751082172.d9e28756f78b0555a6733ffbf6ab98bd., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 because the exception is null or not the one we care about 2024-11-16T09:58:22,067 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9e28756f78b0555a6733ffbf6ab98bd, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T09:58:22,067 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] ipc.CallRunner(138): callId: 196 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39654 deadline: 1731751162063, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9e28756f78b0555a6733ffbf6ab98bd, server=037c58a19ddb,37267,1731751077844 2024-11-16T09:58:22,068 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731751082172.d9e28756f78b0555a6733ffbf6ab98bd., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 , the old value is region=TestAcidGuarantees,,1731751082172.d9e28756f78b0555a6733ffbf6ab98bd., hostname=037c58a19ddb,37267,1731751077844, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9e28756f78b0555a6733ffbf6ab98bd, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:58:22,068 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731751082172.d9e28756f78b0555a6733ffbf6ab98bd., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9e28756f78b0555a6733ffbf6ab98bd, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:58:22,068 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731751082172.d9e28756f78b0555a6733ffbf6ab98bd., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 because the exception is null or not the one we care about 2024-11-16T09:58:22,116 DEBUG [RS:0;037c58a19ddb:37267-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/d9e28756f78b0555a6733ffbf6ab98bd/.tmp/C/95c0d63049154ac794ce470e2eb2ed66 as hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/d9e28756f78b0555a6733ffbf6ab98bd/C/95c0d63049154ac794ce470e2eb2ed66 2024-11-16T09:58:22,122 INFO [RS:0;037c58a19ddb:37267-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in d9e28756f78b0555a6733ffbf6ab98bd/C of d9e28756f78b0555a6733ffbf6ab98bd into 95c0d63049154ac794ce470e2eb2ed66(size=13.1 K), total size for store is 13.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-16T09:58:22,122 DEBUG [RS:0;037c58a19ddb:37267-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for d9e28756f78b0555a6733ffbf6ab98bd: 2024-11-16T09:58:22,122 INFO [RS:0;037c58a19ddb:37267-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731751082172.d9e28756f78b0555a6733ffbf6ab98bd., storeName=d9e28756f78b0555a6733ffbf6ab98bd/C, priority=13, startTime=1731751101625; duration=0sec 2024-11-16T09:58:22,122 DEBUG [RS:0;037c58a19ddb:37267-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-16T09:58:22,122 DEBUG [RS:0;037c58a19ddb:37267-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: d9e28756f78b0555a6733ffbf6ab98bd:C 2024-11-16T09:58:22,273 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9e28756f78b0555a6733ffbf6ab98bd, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T09:58:22,273 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] ipc.CallRunner(138): callId: 197 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39654 deadline: 1731751162272, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9e28756f78b0555a6733ffbf6ab98bd, server=037c58a19ddb,37267,1731751077844 2024-11-16T09:58:22,273 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731751082172.d9e28756f78b0555a6733ffbf6ab98bd., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 , the old value is region=TestAcidGuarantees,,1731751082172.d9e28756f78b0555a6733ffbf6ab98bd., hostname=037c58a19ddb,37267,1731751077844, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9e28756f78b0555a6733ffbf6ab98bd, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:58:22,273 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731751082172.d9e28756f78b0555a6733ffbf6ab98bd., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9e28756f78b0555a6733ffbf6ab98bd, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:58:22,273 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731751082172.d9e28756f78b0555a6733ffbf6ab98bd., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 because the exception is null or not the one we care about 2024-11-16T09:58:22,307 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=509 (bloomFilter=true), to=hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/d9e28756f78b0555a6733ffbf6ab98bd/.tmp/A/2d6792fd6ed14e2abcfec73afc105145 2024-11-16T09:58:22,317 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/d9e28756f78b0555a6733ffbf6ab98bd/.tmp/B/d8e6fbedab5348f9b8cd16f7b9ca02b9 is 50, key is test_row_0/B:col10/1731751101886/Put/seqid=0 2024-11-16T09:58:22,324 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37591 is added to blk_1073741943_1119 (size=9857) 2024-11-16T09:58:22,553 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37877 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=19 2024-11-16T09:58:22,553 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-16T09:58:22,554 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37877 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-16T09:58:22,555 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37877 {}] procedure2.ProcedureExecutor(1139): Stored pid=21, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=21, table=TestAcidGuarantees 2024-11-16T09:58:22,556 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37877 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=21 2024-11-16T09:58:22,556 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=21, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=21, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-16T09:58:22,557 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=21, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=21, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-16T09:58:22,557 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=22, ppid=21, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-16T09:58:22,583 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9e28756f78b0555a6733ffbf6ab98bd, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T09:58:22,584 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] ipc.CallRunner(138): callId: 198 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39654 deadline: 1731751162583, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9e28756f78b0555a6733ffbf6ab98bd, server=037c58a19ddb,37267,1731751077844 2024-11-16T09:58:22,584 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731751082172.d9e28756f78b0555a6733ffbf6ab98bd., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 , the old value is region=TestAcidGuarantees,,1731751082172.d9e28756f78b0555a6733ffbf6ab98bd., hostname=037c58a19ddb,37267,1731751077844, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9e28756f78b0555a6733ffbf6ab98bd, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:58:22,584 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731751082172.d9e28756f78b0555a6733ffbf6ab98bd., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9e28756f78b0555a6733ffbf6ab98bd, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:58:22,584 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731751082172.d9e28756f78b0555a6733ffbf6ab98bd., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 because the exception is null or not the one we care about 2024-11-16T09:58:22,633 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9e28756f78b0555a6733ffbf6ab98bd, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T09:58:22,634 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] ipc.CallRunner(138): callId: 137 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39620 deadline: 1731751162633, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9e28756f78b0555a6733ffbf6ab98bd, server=037c58a19ddb,37267,1731751077844 2024-11-16T09:58:22,634 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9e28756f78b0555a6733ffbf6ab98bd, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T09:58:22,634 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] ipc.CallRunner(138): callId: 130 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39638 deadline: 1731751162633, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9e28756f78b0555a6733ffbf6ab98bd, server=037c58a19ddb,37267,1731751077844 2024-11-16T09:58:22,636 WARN [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRpcRetryingCaller(177): Call to 037c58a19ddb,37267,1731751077844 for 'test_row_1' in d9e28756f78b0555a6733ffbf6ab98bd of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4177 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9e28756f78b0555a6733ffbf6ab98bd, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor44.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9e28756f78b0555a6733ffbf6ab98bd, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-16T09:58:22,637 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731751082172.d9e28756f78b0555a6733ffbf6ab98bd., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 , the old value is region=TestAcidGuarantees,,1731751082172.d9e28756f78b0555a6733ffbf6ab98bd., hostname=037c58a19ddb,37267,1731751077844, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9e28756f78b0555a6733ffbf6ab98bd, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:58:22,637 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731751082172.d9e28756f78b0555a6733ffbf6ab98bd., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9e28756f78b0555a6733ffbf6ab98bd, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:58:22,637 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731751082172.d9e28756f78b0555a6733ffbf6ab98bd., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 because the exception is null or not the one we care about 2024-11-16T09:58:22,637 WARN [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRpcRetryingCaller(177): Call to 037c58a19ddb,37267,1731751077844 for 'test_row_0' in d9e28756f78b0555a6733ffbf6ab98bd of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4173 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9e28756f78b0555a6733ffbf6ab98bd, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor44.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9e28756f78b0555a6733ffbf6ab98bd, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-16T09:58:22,638 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731751082172.d9e28756f78b0555a6733ffbf6ab98bd., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 , the old value is region=TestAcidGuarantees,,1731751082172.d9e28756f78b0555a6733ffbf6ab98bd., hostname=037c58a19ddb,37267,1731751077844, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9e28756f78b0555a6733ffbf6ab98bd, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:58:22,638 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731751082172.d9e28756f78b0555a6733ffbf6ab98bd., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9e28756f78b0555a6733ffbf6ab98bd, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:58:22,638 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731751082172.d9e28756f78b0555a6733ffbf6ab98bd., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 because the exception is null or not the one we care about 2024-11-16T09:58:22,644 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9e28756f78b0555a6733ffbf6ab98bd, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T09:58:22,644 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] ipc.CallRunner(138): callId: 134 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39630 deadline: 1731751162643, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9e28756f78b0555a6733ffbf6ab98bd, server=037c58a19ddb,37267,1731751077844 2024-11-16T09:58:22,644 WARN [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRpcRetryingCaller(177): Call to 037c58a19ddb,37267,1731751077844 for 'test_row_2' in d9e28756f78b0555a6733ffbf6ab98bd of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4185 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9e28756f78b0555a6733ffbf6ab98bd, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor44.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9e28756f78b0555a6733ffbf6ab98bd, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-16T09:58:22,645 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731751082172.d9e28756f78b0555a6733ffbf6ab98bd., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 , the old value is region=TestAcidGuarantees,,1731751082172.d9e28756f78b0555a6733ffbf6ab98bd., hostname=037c58a19ddb,37267,1731751077844, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9e28756f78b0555a6733ffbf6ab98bd, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:58:22,645 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731751082172.d9e28756f78b0555a6733ffbf6ab98bd., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9e28756f78b0555a6733ffbf6ab98bd, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:58:22,645 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731751082172.d9e28756f78b0555a6733ffbf6ab98bd., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 because the exception is null or not the one we care about 2024-11-16T09:58:22,654 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9e28756f78b0555a6733ffbf6ab98bd, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T09:58:22,654 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] ipc.CallRunner(138): callId: 135 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39696 deadline: 1731751162653, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9e28756f78b0555a6733ffbf6ab98bd, server=037c58a19ddb,37267,1731751077844 2024-11-16T09:58:22,654 WARN [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRpcRetryingCaller(177): Call to 037c58a19ddb,37267,1731751077844 for 'test_row_0' in d9e28756f78b0555a6733ffbf6ab98bd of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4193 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9e28756f78b0555a6733ffbf6ab98bd, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor44.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9e28756f78b0555a6733ffbf6ab98bd, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-16T09:58:22,655 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731751082172.d9e28756f78b0555a6733ffbf6ab98bd., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 , the old value is region=TestAcidGuarantees,,1731751082172.d9e28756f78b0555a6733ffbf6ab98bd., hostname=037c58a19ddb,37267,1731751077844, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9e28756f78b0555a6733ffbf6ab98bd, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:58:22,655 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731751082172.d9e28756f78b0555a6733ffbf6ab98bd., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9e28756f78b0555a6733ffbf6ab98bd, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:58:22,655 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731751082172.d9e28756f78b0555a6733ffbf6ab98bd., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 because the exception is null or not the one we care about 2024-11-16T09:58:22,663 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37877 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=21 2024-11-16T09:58:22,710 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=37267 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=22 2024-11-16T09:58:22,710 DEBUG [RS_FLUSH_OPERATIONS-regionserver/037c58a19ddb:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731751082172.d9e28756f78b0555a6733ffbf6ab98bd. 2024-11-16T09:58:22,710 DEBUG [RS_FLUSH_OPERATIONS-regionserver/037c58a19ddb:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731751082172.d9e28756f78b0555a6733ffbf6ab98bd. as already flushing 2024-11-16T09:58:22,711 DEBUG [RS_FLUSH_OPERATIONS-regionserver/037c58a19ddb:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731751082172.d9e28756f78b0555a6733ffbf6ab98bd. 2024-11-16T09:58:22,711 ERROR [RS_FLUSH_OPERATIONS-regionserver/037c58a19ddb:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] handler.RSProcedureHandler(58): pid=22 java.io.IOException: Unable to complete flush {ENCODED => d9e28756f78b0555a6733ffbf6ab98bd, NAME => 'TestAcidGuarantees,,1731751082172.d9e28756f78b0555a6733ffbf6ab98bd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T09:58:22,711 DEBUG [RS_FLUSH_OPERATIONS-regionserver/037c58a19ddb:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=22 java.io.IOException: Unable to complete flush {ENCODED => d9e28756f78b0555a6733ffbf6ab98bd, NAME => 'TestAcidGuarantees,,1731751082172.d9e28756f78b0555a6733ffbf6ab98bd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T09:58:22,711 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37877 {}] master.HMaster(4177): Remote procedure failed, pid=22 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d9e28756f78b0555a6733ffbf6ab98bd, NAME => 'TestAcidGuarantees,,1731751082172.d9e28756f78b0555a6733ffbf6ab98bd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d9e28756f78b0555a6733ffbf6ab98bd, NAME => 'TestAcidGuarantees,,1731751082172.d9e28756f78b0555a6733ffbf6ab98bd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T09:58:22,723 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=509 (bloomFilter=true), to=hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/d9e28756f78b0555a6733ffbf6ab98bd/.tmp/B/d8e6fbedab5348f9b8cd16f7b9ca02b9 2024-11-16T09:58:22,732 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/d9e28756f78b0555a6733ffbf6ab98bd/.tmp/C/3cfb8ef2898847a385d61a4e27bcc4b7 is 50, key is test_row_0/C:col10/1731751101886/Put/seqid=0 2024-11-16T09:58:22,740 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37591 is added to blk_1073741944_1120 (size=9857) 2024-11-16T09:58:22,740 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=509 (bloomFilter=true), to=hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/d9e28756f78b0555a6733ffbf6ab98bd/.tmp/C/3cfb8ef2898847a385d61a4e27bcc4b7 2024-11-16T09:58:22,747 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/d9e28756f78b0555a6733ffbf6ab98bd/.tmp/A/2d6792fd6ed14e2abcfec73afc105145 as hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/d9e28756f78b0555a6733ffbf6ab98bd/A/2d6792fd6ed14e2abcfec73afc105145 2024-11-16T09:58:22,754 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/d9e28756f78b0555a6733ffbf6ab98bd/A/2d6792fd6ed14e2abcfec73afc105145, entries=100, sequenceid=509, filesize=9.6 K 2024-11-16T09:58:22,756 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/d9e28756f78b0555a6733ffbf6ab98bd/.tmp/B/d8e6fbedab5348f9b8cd16f7b9ca02b9 as hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/d9e28756f78b0555a6733ffbf6ab98bd/B/d8e6fbedab5348f9b8cd16f7b9ca02b9 2024-11-16T09:58:22,761 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/d9e28756f78b0555a6733ffbf6ab98bd/B/d8e6fbedab5348f9b8cd16f7b9ca02b9, entries=100, sequenceid=509, filesize=9.6 K 2024-11-16T09:58:22,762 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/d9e28756f78b0555a6733ffbf6ab98bd/.tmp/C/3cfb8ef2898847a385d61a4e27bcc4b7 as hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/d9e28756f78b0555a6733ffbf6ab98bd/C/3cfb8ef2898847a385d61a4e27bcc4b7 2024-11-16T09:58:22,767 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/d9e28756f78b0555a6733ffbf6ab98bd/C/3cfb8ef2898847a385d61a4e27bcc4b7, entries=100, sequenceid=509, filesize=9.6 K 2024-11-16T09:58:22,768 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~141.33 KB/144720, currentSize=147.60 KB/151140 for d9e28756f78b0555a6733ffbf6ab98bd in 877ms, sequenceid=509, compaction requested=false 2024-11-16T09:58:22,768 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for d9e28756f78b0555a6733ffbf6ab98bd: 2024-11-16T09:58:22,864 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=37267 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=22 2024-11-16T09:58:22,864 DEBUG [RS_FLUSH_OPERATIONS-regionserver/037c58a19ddb:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731751082172.d9e28756f78b0555a6733ffbf6ab98bd. 2024-11-16T09:58:22,865 INFO [RS_FLUSH_OPERATIONS-regionserver/037c58a19ddb:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HRegion(2902): Flushing d9e28756f78b0555a6733ffbf6ab98bd 3/3 column families, dataSize=147.60 KB heapSize=387.47 KB 2024-11-16T09:58:22,873 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37877 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=21 2024-11-16T09:58:22,874 DEBUG [RS_FLUSH_OPERATIONS-regionserver/037c58a19ddb:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/d9e28756f78b0555a6733ffbf6ab98bd/.tmp/A/62cf785409214e10b9be44024b7bff9b is 50, key is test_row_0/A:col10/1731751101943/Put/seqid=0 2024-11-16T09:58:22,882 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37591 is added to blk_1073741945_1121 (size=12301) 2024-11-16T09:58:22,883 INFO [RS_FLUSH_OPERATIONS-regionserver/037c58a19ddb:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=535 (bloomFilter=true), to=hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/d9e28756f78b0555a6733ffbf6ab98bd/.tmp/A/62cf785409214e10b9be44024b7bff9b 2024-11-16T09:58:22,898 DEBUG [RS_FLUSH_OPERATIONS-regionserver/037c58a19ddb:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/d9e28756f78b0555a6733ffbf6ab98bd/.tmp/B/7ad96d41e26d439699f66cb611331ec1 is 50, key is test_row_0/B:col10/1731751101943/Put/seqid=0 2024-11-16T09:58:22,905 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37591 is added to blk_1073741946_1122 (size=12301) 2024-11-16T09:58:22,906 INFO [RS_FLUSH_OPERATIONS-regionserver/037c58a19ddb:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=535 (bloomFilter=true), to=hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/d9e28756f78b0555a6733ffbf6ab98bd/.tmp/B/7ad96d41e26d439699f66cb611331ec1 2024-11-16T09:58:22,917 DEBUG [RS_FLUSH_OPERATIONS-regionserver/037c58a19ddb:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/d9e28756f78b0555a6733ffbf6ab98bd/.tmp/C/6513ccc313dc49b28664372e3f14691a is 50, key is test_row_0/C:col10/1731751101943/Put/seqid=0 2024-11-16T09:58:22,924 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37591 is added to blk_1073741947_1123 (size=12301) 2024-11-16T09:58:22,929 INFO [RS_FLUSH_OPERATIONS-regionserver/037c58a19ddb:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=535 (bloomFilter=true), to=hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/d9e28756f78b0555a6733ffbf6ab98bd/.tmp/C/6513ccc313dc49b28664372e3f14691a 2024-11-16T09:58:22,935 DEBUG [RS_FLUSH_OPERATIONS-regionserver/037c58a19ddb:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/d9e28756f78b0555a6733ffbf6ab98bd/.tmp/A/62cf785409214e10b9be44024b7bff9b as hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/d9e28756f78b0555a6733ffbf6ab98bd/A/62cf785409214e10b9be44024b7bff9b 2024-11-16T09:58:22,940 INFO [RS_FLUSH_OPERATIONS-regionserver/037c58a19ddb:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/d9e28756f78b0555a6733ffbf6ab98bd/A/62cf785409214e10b9be44024b7bff9b, entries=150, sequenceid=535, filesize=12.0 K 2024-11-16T09:58:22,941 DEBUG [RS_FLUSH_OPERATIONS-regionserver/037c58a19ddb:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/d9e28756f78b0555a6733ffbf6ab98bd/.tmp/B/7ad96d41e26d439699f66cb611331ec1 as hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/d9e28756f78b0555a6733ffbf6ab98bd/B/7ad96d41e26d439699f66cb611331ec1 2024-11-16T09:58:22,947 INFO [RS_FLUSH_OPERATIONS-regionserver/037c58a19ddb:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/d9e28756f78b0555a6733ffbf6ab98bd/B/7ad96d41e26d439699f66cb611331ec1, entries=150, sequenceid=535, filesize=12.0 K 2024-11-16T09:58:22,950 DEBUG [RS_FLUSH_OPERATIONS-regionserver/037c58a19ddb:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/d9e28756f78b0555a6733ffbf6ab98bd/.tmp/C/6513ccc313dc49b28664372e3f14691a as hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/d9e28756f78b0555a6733ffbf6ab98bd/C/6513ccc313dc49b28664372e3f14691a 2024-11-16T09:58:22,960 INFO [RS_FLUSH_OPERATIONS-regionserver/037c58a19ddb:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/d9e28756f78b0555a6733ffbf6ab98bd/C/6513ccc313dc49b28664372e3f14691a, entries=150, sequenceid=535, filesize=12.0 K 2024-11-16T09:58:22,962 INFO [RS_FLUSH_OPERATIONS-regionserver/037c58a19ddb:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HRegion(3140): Finished flush of dataSize ~147.60 KB/151140, heapSize ~387.42 KB/396720, currentSize=0 B/0 for d9e28756f78b0555a6733ffbf6ab98bd in 97ms, sequenceid=535, compaction requested=true 2024-11-16T09:58:22,962 DEBUG [RS_FLUSH_OPERATIONS-regionserver/037c58a19ddb:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HRegion(2603): Flush status journal for d9e28756f78b0555a6733ffbf6ab98bd: 2024-11-16T09:58:22,962 DEBUG [RS_FLUSH_OPERATIONS-regionserver/037c58a19ddb:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731751082172.d9e28756f78b0555a6733ffbf6ab98bd. 2024-11-16T09:58:22,962 DEBUG [RS_FLUSH_OPERATIONS-regionserver/037c58a19ddb:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=22 2024-11-16T09:58:22,962 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37877 {}] master.HMaster(4169): Remote procedure done, pid=22 2024-11-16T09:58:22,966 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=22, resume processing ppid=21 2024-11-16T09:58:22,966 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=22, ppid=21, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 407 msec 2024-11-16T09:58:22,968 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=21, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=21, table=TestAcidGuarantees in 413 msec 2024-11-16T09:58:23,127 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] regionserver.HRegion(8855): Flush requested on d9e28756f78b0555a6733ffbf6ab98bd 2024-11-16T09:58:23,128 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-16T09:58:23,128 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing d9e28756f78b0555a6733ffbf6ab98bd 3/3 column families, dataSize=53.67 KB heapSize=141.38 KB 2024-11-16T09:58:23,134 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/d9e28756f78b0555a6733ffbf6ab98bd/.tmp/A/984513ede0154fe5b72bad83157b16af is 50, key is test_row_0/A:col10/1731751103094/Put/seqid=0 2024-11-16T09:58:23,146 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37591 is added to blk_1073741948_1124 (size=14741) 2024-11-16T09:58:23,183 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37877 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=21 2024-11-16T09:58:23,183 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-16T09:58:23,184 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37877 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-16T09:58:23,185 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37877 {}] procedure2.ProcedureExecutor(1139): Stored pid=23, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=23, table=TestAcidGuarantees 2024-11-16T09:58:23,186 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37877 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=23 2024-11-16T09:58:23,186 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=23, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=23, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-16T09:58:23,187 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=23, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=23, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-16T09:58:23,187 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=24, ppid=23, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-16T09:58:23,212 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9e28756f78b0555a6733ffbf6ab98bd, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T09:58:23,212 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] ipc.CallRunner(138): callId: 229 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39654 deadline: 1731751163208, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9e28756f78b0555a6733ffbf6ab98bd, server=037c58a19ddb,37267,1731751077844 2024-11-16T09:58:23,213 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731751082172.d9e28756f78b0555a6733ffbf6ab98bd., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 , the old value is region=TestAcidGuarantees,,1731751082172.d9e28756f78b0555a6733ffbf6ab98bd., hostname=037c58a19ddb,37267,1731751077844, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9e28756f78b0555a6733ffbf6ab98bd, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:58:23,213 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731751082172.d9e28756f78b0555a6733ffbf6ab98bd., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9e28756f78b0555a6733ffbf6ab98bd, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:58:23,213 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731751082172.d9e28756f78b0555a6733ffbf6ab98bd., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 because the exception is null or not the one we care about 2024-11-16T09:58:23,293 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37877 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=23 2024-11-16T09:58:23,327 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9e28756f78b0555a6733ffbf6ab98bd, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T09:58:23,327 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] ipc.CallRunner(138): callId: 230 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39654 deadline: 1731751163323, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9e28756f78b0555a6733ffbf6ab98bd, server=037c58a19ddb,37267,1731751077844 2024-11-16T09:58:23,328 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731751082172.d9e28756f78b0555a6733ffbf6ab98bd., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 , the old value is region=TestAcidGuarantees,,1731751082172.d9e28756f78b0555a6733ffbf6ab98bd., hostname=037c58a19ddb,37267,1731751077844, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9e28756f78b0555a6733ffbf6ab98bd, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:58:23,328 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731751082172.d9e28756f78b0555a6733ffbf6ab98bd., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9e28756f78b0555a6733ffbf6ab98bd, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:58:23,328 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731751082172.d9e28756f78b0555a6733ffbf6ab98bd., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 because the exception is null or not the one we care about 2024-11-16T09:58:23,341 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=37267 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=24 2024-11-16T09:58:23,341 DEBUG [RS_FLUSH_OPERATIONS-regionserver/037c58a19ddb:0-1 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731751082172.d9e28756f78b0555a6733ffbf6ab98bd. 2024-11-16T09:58:23,342 DEBUG [RS_FLUSH_OPERATIONS-regionserver/037c58a19ddb:0-1 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731751082172.d9e28756f78b0555a6733ffbf6ab98bd. as already flushing 2024-11-16T09:58:23,342 DEBUG [RS_FLUSH_OPERATIONS-regionserver/037c58a19ddb:0-1 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731751082172.d9e28756f78b0555a6733ffbf6ab98bd. 2024-11-16T09:58:23,342 ERROR [RS_FLUSH_OPERATIONS-regionserver/037c58a19ddb:0-1 {event_type=RS_FLUSH_REGIONS, pid=24}] handler.RSProcedureHandler(58): pid=24 java.io.IOException: Unable to complete flush {ENCODED => d9e28756f78b0555a6733ffbf6ab98bd, NAME => 'TestAcidGuarantees,,1731751082172.d9e28756f78b0555a6733ffbf6ab98bd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T09:58:23,342 DEBUG [RS_FLUSH_OPERATIONS-regionserver/037c58a19ddb:0-1 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=24 java.io.IOException: Unable to complete flush {ENCODED => d9e28756f78b0555a6733ffbf6ab98bd, NAME => 'TestAcidGuarantees,,1731751082172.d9e28756f78b0555a6733ffbf6ab98bd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T09:58:23,342 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37877 {}] master.HMaster(4177): Remote procedure failed, pid=24 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d9e28756f78b0555a6733ffbf6ab98bd, NAME => 'TestAcidGuarantees,,1731751082172.d9e28756f78b0555a6733ffbf6ab98bd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d9e28756f78b0555a6733ffbf6ab98bd, NAME => 'TestAcidGuarantees,,1731751082172.d9e28756f78b0555a6733ffbf6ab98bd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T09:58:23,495 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=37267 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=24 2024-11-16T09:58:23,495 DEBUG [RS_FLUSH_OPERATIONS-regionserver/037c58a19ddb:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731751082172.d9e28756f78b0555a6733ffbf6ab98bd. 2024-11-16T09:58:23,495 DEBUG [RS_FLUSH_OPERATIONS-regionserver/037c58a19ddb:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731751082172.d9e28756f78b0555a6733ffbf6ab98bd. as already flushing 2024-11-16T09:58:23,495 DEBUG [RS_FLUSH_OPERATIONS-regionserver/037c58a19ddb:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731751082172.d9e28756f78b0555a6733ffbf6ab98bd. 2024-11-16T09:58:23,495 ERROR [RS_FLUSH_OPERATIONS-regionserver/037c58a19ddb:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] handler.RSProcedureHandler(58): pid=24 java.io.IOException: Unable to complete flush {ENCODED => d9e28756f78b0555a6733ffbf6ab98bd, NAME => 'TestAcidGuarantees,,1731751082172.d9e28756f78b0555a6733ffbf6ab98bd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T09:58:23,496 DEBUG [RS_FLUSH_OPERATIONS-regionserver/037c58a19ddb:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=24 java.io.IOException: Unable to complete flush {ENCODED => d9e28756f78b0555a6733ffbf6ab98bd, NAME => 'TestAcidGuarantees,,1731751082172.d9e28756f78b0555a6733ffbf6ab98bd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T09:58:23,496 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37877 {}] master.HMaster(4177): Remote procedure failed, pid=24 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d9e28756f78b0555a6733ffbf6ab98bd, NAME => 'TestAcidGuarantees,,1731751082172.d9e28756f78b0555a6733ffbf6ab98bd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d9e28756f78b0555a6733ffbf6ab98bd, NAME => 'TestAcidGuarantees,,1731751082172.d9e28756f78b0555a6733ffbf6ab98bd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T09:58:23,503 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37877 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=23 2024-11-16T09:58:23,535 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9e28756f78b0555a6733ffbf6ab98bd, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T09:58:23,535 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] ipc.CallRunner(138): callId: 231 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39654 deadline: 1731751163533, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9e28756f78b0555a6733ffbf6ab98bd, server=037c58a19ddb,37267,1731751077844 2024-11-16T09:58:23,536 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731751082172.d9e28756f78b0555a6733ffbf6ab98bd., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 , the old value is region=TestAcidGuarantees,,1731751082172.d9e28756f78b0555a6733ffbf6ab98bd., hostname=037c58a19ddb,37267,1731751077844, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9e28756f78b0555a6733ffbf6ab98bd, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:58:23,536 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731751082172.d9e28756f78b0555a6733ffbf6ab98bd., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9e28756f78b0555a6733ffbf6ab98bd, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:58:23,536 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731751082172.d9e28756f78b0555a6733ffbf6ab98bd., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 because the exception is null or not the one we care about 2024-11-16T09:58:23,547 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=546 (bloomFilter=true), to=hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/d9e28756f78b0555a6733ffbf6ab98bd/.tmp/A/984513ede0154fe5b72bad83157b16af 2024-11-16T09:58:23,557 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/d9e28756f78b0555a6733ffbf6ab98bd/.tmp/B/e600d8b124904d24bf48bfa690411916 is 50, key is test_row_0/B:col10/1731751103094/Put/seqid=0 2024-11-16T09:58:23,561 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37591 is added to blk_1073741949_1125 (size=12301) 2024-11-16T09:58:23,648 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=37267 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=24 2024-11-16T09:58:23,649 DEBUG [RS_FLUSH_OPERATIONS-regionserver/037c58a19ddb:0-0 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731751082172.d9e28756f78b0555a6733ffbf6ab98bd. 2024-11-16T09:58:23,649 DEBUG [RS_FLUSH_OPERATIONS-regionserver/037c58a19ddb:0-0 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731751082172.d9e28756f78b0555a6733ffbf6ab98bd. as already flushing 2024-11-16T09:58:23,649 DEBUG [RS_FLUSH_OPERATIONS-regionserver/037c58a19ddb:0-0 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731751082172.d9e28756f78b0555a6733ffbf6ab98bd. 2024-11-16T09:58:23,649 ERROR [RS_FLUSH_OPERATIONS-regionserver/037c58a19ddb:0-0 {event_type=RS_FLUSH_REGIONS, pid=24}] handler.RSProcedureHandler(58): pid=24 java.io.IOException: Unable to complete flush {ENCODED => d9e28756f78b0555a6733ffbf6ab98bd, NAME => 'TestAcidGuarantees,,1731751082172.d9e28756f78b0555a6733ffbf6ab98bd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T09:58:23,649 DEBUG [RS_FLUSH_OPERATIONS-regionserver/037c58a19ddb:0-0 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=24 java.io.IOException: Unable to complete flush {ENCODED => d9e28756f78b0555a6733ffbf6ab98bd, NAME => 'TestAcidGuarantees,,1731751082172.d9e28756f78b0555a6733ffbf6ab98bd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T09:58:23,650 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37877 {}] master.HMaster(4177): Remote procedure failed, pid=24 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d9e28756f78b0555a6733ffbf6ab98bd, NAME => 'TestAcidGuarantees,,1731751082172.d9e28756f78b0555a6733ffbf6ab98bd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d9e28756f78b0555a6733ffbf6ab98bd, NAME => 'TestAcidGuarantees,,1731751082172.d9e28756f78b0555a6733ffbf6ab98bd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T09:58:23,802 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=37267 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=24 2024-11-16T09:58:23,802 DEBUG [RS_FLUSH_OPERATIONS-regionserver/037c58a19ddb:0-1 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731751082172.d9e28756f78b0555a6733ffbf6ab98bd. 2024-11-16T09:58:23,802 DEBUG [RS_FLUSH_OPERATIONS-regionserver/037c58a19ddb:0-1 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731751082172.d9e28756f78b0555a6733ffbf6ab98bd. as already flushing 2024-11-16T09:58:23,802 DEBUG [RS_FLUSH_OPERATIONS-regionserver/037c58a19ddb:0-1 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731751082172.d9e28756f78b0555a6733ffbf6ab98bd. 2024-11-16T09:58:23,803 ERROR [RS_FLUSH_OPERATIONS-regionserver/037c58a19ddb:0-1 {event_type=RS_FLUSH_REGIONS, pid=24}] handler.RSProcedureHandler(58): pid=24 java.io.IOException: Unable to complete flush {ENCODED => d9e28756f78b0555a6733ffbf6ab98bd, NAME => 'TestAcidGuarantees,,1731751082172.d9e28756f78b0555a6733ffbf6ab98bd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T09:58:23,803 DEBUG [RS_FLUSH_OPERATIONS-regionserver/037c58a19ddb:0-1 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=24 java.io.IOException: Unable to complete flush {ENCODED => d9e28756f78b0555a6733ffbf6ab98bd, NAME => 'TestAcidGuarantees,,1731751082172.d9e28756f78b0555a6733ffbf6ab98bd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T09:58:23,803 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37877 {}] master.HMaster(4177): Remote procedure failed, pid=24 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d9e28756f78b0555a6733ffbf6ab98bd, NAME => 'TestAcidGuarantees,,1731751082172.d9e28756f78b0555a6733ffbf6ab98bd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d9e28756f78b0555a6733ffbf6ab98bd, NAME => 'TestAcidGuarantees,,1731751082172.d9e28756f78b0555a6733ffbf6ab98bd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T09:58:23,813 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37877 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=23 2024-11-16T09:58:23,845 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9e28756f78b0555a6733ffbf6ab98bd, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T09:58:23,845 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] ipc.CallRunner(138): callId: 232 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39654 deadline: 1731751163842, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9e28756f78b0555a6733ffbf6ab98bd, server=037c58a19ddb,37267,1731751077844 2024-11-16T09:58:23,845 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731751082172.d9e28756f78b0555a6733ffbf6ab98bd., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 , the old value is region=TestAcidGuarantees,,1731751082172.d9e28756f78b0555a6733ffbf6ab98bd., hostname=037c58a19ddb,37267,1731751077844, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9e28756f78b0555a6733ffbf6ab98bd, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:58:23,845 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731751082172.d9e28756f78b0555a6733ffbf6ab98bd., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9e28756f78b0555a6733ffbf6ab98bd, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:58:23,845 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731751082172.d9e28756f78b0555a6733ffbf6ab98bd., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 because the exception is null or not the one we care about 2024-11-16T09:58:23,956 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=37267 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=24 2024-11-16T09:58:23,956 DEBUG [RS_FLUSH_OPERATIONS-regionserver/037c58a19ddb:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731751082172.d9e28756f78b0555a6733ffbf6ab98bd. 2024-11-16T09:58:23,956 DEBUG [RS_FLUSH_OPERATIONS-regionserver/037c58a19ddb:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731751082172.d9e28756f78b0555a6733ffbf6ab98bd. as already flushing 2024-11-16T09:58:23,956 DEBUG [RS_FLUSH_OPERATIONS-regionserver/037c58a19ddb:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731751082172.d9e28756f78b0555a6733ffbf6ab98bd. 2024-11-16T09:58:23,956 ERROR [RS_FLUSH_OPERATIONS-regionserver/037c58a19ddb:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] handler.RSProcedureHandler(58): pid=24 java.io.IOException: Unable to complete flush {ENCODED => d9e28756f78b0555a6733ffbf6ab98bd, NAME => 'TestAcidGuarantees,,1731751082172.d9e28756f78b0555a6733ffbf6ab98bd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T09:58:23,956 DEBUG [RS_FLUSH_OPERATIONS-regionserver/037c58a19ddb:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=24 java.io.IOException: Unable to complete flush {ENCODED => d9e28756f78b0555a6733ffbf6ab98bd, NAME => 'TestAcidGuarantees,,1731751082172.d9e28756f78b0555a6733ffbf6ab98bd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T09:58:23,957 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37877 {}] master.HMaster(4177): Remote procedure failed, pid=24 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d9e28756f78b0555a6733ffbf6ab98bd, NAME => 'TestAcidGuarantees,,1731751082172.d9e28756f78b0555a6733ffbf6ab98bd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d9e28756f78b0555a6733ffbf6ab98bd, NAME => 'TestAcidGuarantees,,1731751082172.d9e28756f78b0555a6733ffbf6ab98bd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T09:58:23,962 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=546 (bloomFilter=true), to=hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/d9e28756f78b0555a6733ffbf6ab98bd/.tmp/B/e600d8b124904d24bf48bfa690411916 2024-11-16T09:58:23,973 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/d9e28756f78b0555a6733ffbf6ab98bd/.tmp/C/a740d4a75fb5467784531979bb9dda59 is 50, key is test_row_0/C:col10/1731751103094/Put/seqid=0 2024-11-16T09:58:23,981 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37591 is added to blk_1073741950_1126 (size=12301) 2024-11-16T09:58:23,982 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=546 (bloomFilter=true), to=hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/d9e28756f78b0555a6733ffbf6ab98bd/.tmp/C/a740d4a75fb5467784531979bb9dda59 2024-11-16T09:58:23,988 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/d9e28756f78b0555a6733ffbf6ab98bd/.tmp/A/984513ede0154fe5b72bad83157b16af as hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/d9e28756f78b0555a6733ffbf6ab98bd/A/984513ede0154fe5b72bad83157b16af 2024-11-16T09:58:23,997 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/d9e28756f78b0555a6733ffbf6ab98bd/A/984513ede0154fe5b72bad83157b16af, entries=200, sequenceid=546, filesize=14.4 K 2024-11-16T09:58:23,998 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/d9e28756f78b0555a6733ffbf6ab98bd/.tmp/B/e600d8b124904d24bf48bfa690411916 as hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/d9e28756f78b0555a6733ffbf6ab98bd/B/e600d8b124904d24bf48bfa690411916 2024-11-16T09:58:24,002 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/d9e28756f78b0555a6733ffbf6ab98bd/B/e600d8b124904d24bf48bfa690411916, entries=150, sequenceid=546, filesize=12.0 K 2024-11-16T09:58:24,004 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/d9e28756f78b0555a6733ffbf6ab98bd/.tmp/C/a740d4a75fb5467784531979bb9dda59 as hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/d9e28756f78b0555a6733ffbf6ab98bd/C/a740d4a75fb5467784531979bb9dda59 2024-11-16T09:58:24,009 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/d9e28756f78b0555a6733ffbf6ab98bd/C/a740d4a75fb5467784531979bb9dda59, entries=150, sequenceid=546, filesize=12.0 K 2024-11-16T09:58:24,010 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~141.33 KB/144720, currentSize=147.60 KB/151140 for d9e28756f78b0555a6733ffbf6ab98bd in 882ms, sequenceid=546, compaction requested=true 2024-11-16T09:58:24,010 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for d9e28756f78b0555a6733ffbf6ab98bd: 2024-11-16T09:58:24,010 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store d9e28756f78b0555a6733ffbf6ab98bd:A, priority=-2147483648, current under compaction store size is 1 2024-11-16T09:58:24,010 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-16T09:58:24,010 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store d9e28756f78b0555a6733ffbf6ab98bd:B, priority=-2147483648, current under compaction store size is 2 2024-11-16T09:58:24,010 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-16T09:58:24,010 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store d9e28756f78b0555a6733ffbf6ab98bd:C, priority=-2147483648, current under compaction store size is 3 2024-11-16T09:58:24,010 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-16T09:58:24,010 DEBUG [RS:0;037c58a19ddb:37267-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-16T09:58:24,010 DEBUG [RS:0;037c58a19ddb:37267-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-16T09:58:24,012 DEBUG [RS:0;037c58a19ddb:37267-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 47918 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-16T09:58:24,012 DEBUG [RS:0;037c58a19ddb:37267-longCompactions-0 {}] regionserver.HStore(1541): d9e28756f78b0555a6733ffbf6ab98bd/B is initiating minor compaction (all files) 2024-11-16T09:58:24,012 INFO [RS:0;037c58a19ddb:37267-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of d9e28756f78b0555a6733ffbf6ab98bd/B in TestAcidGuarantees,,1731751082172.d9e28756f78b0555a6733ffbf6ab98bd. 2024-11-16T09:58:24,012 INFO [RS:0;037c58a19ddb:37267-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/d9e28756f78b0555a6733ffbf6ab98bd/B/bc86639a0a354f0aaec9fc1c395896b8, hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/d9e28756f78b0555a6733ffbf6ab98bd/B/d8e6fbedab5348f9b8cd16f7b9ca02b9, hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/d9e28756f78b0555a6733ffbf6ab98bd/B/7ad96d41e26d439699f66cb611331ec1, hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/d9e28756f78b0555a6733ffbf6ab98bd/B/e600d8b124904d24bf48bfa690411916] into tmpdir=hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/d9e28756f78b0555a6733ffbf6ab98bd/.tmp, totalSize=46.8 K 2024-11-16T09:58:24,012 DEBUG [RS:0;037c58a19ddb:37267-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 50358 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-16T09:58:24,012 DEBUG [RS:0;037c58a19ddb:37267-shortCompactions-0 {}] regionserver.HStore(1541): d9e28756f78b0555a6733ffbf6ab98bd/A is initiating minor compaction (all files) 2024-11-16T09:58:24,012 DEBUG [RS:0;037c58a19ddb:37267-longCompactions-0 {}] compactions.Compactor(225): Compacting bc86639a0a354f0aaec9fc1c395896b8, keycount=150, bloomtype=ROW, size=13.1 K, encoding=NONE, compression=NONE, seqNum=496, earliestPutTs=1731751100079 2024-11-16T09:58:24,012 INFO [RS:0;037c58a19ddb:37267-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of d9e28756f78b0555a6733ffbf6ab98bd/A in TestAcidGuarantees,,1731751082172.d9e28756f78b0555a6733ffbf6ab98bd. 2024-11-16T09:58:24,013 INFO [RS:0;037c58a19ddb:37267-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/d9e28756f78b0555a6733ffbf6ab98bd/A/f84ef8c17537456cb6d451fa35e32d14, hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/d9e28756f78b0555a6733ffbf6ab98bd/A/2d6792fd6ed14e2abcfec73afc105145, hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/d9e28756f78b0555a6733ffbf6ab98bd/A/62cf785409214e10b9be44024b7bff9b, hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/d9e28756f78b0555a6733ffbf6ab98bd/A/984513ede0154fe5b72bad83157b16af] into tmpdir=hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/d9e28756f78b0555a6733ffbf6ab98bd/.tmp, totalSize=49.2 K 2024-11-16T09:58:24,013 DEBUG [RS:0;037c58a19ddb:37267-longCompactions-0 {}] compactions.Compactor(225): Compacting d8e6fbedab5348f9b8cd16f7b9ca02b9, keycount=100, bloomtype=ROW, size=9.6 K, encoding=NONE, compression=NONE, seqNum=509, earliestPutTs=1731751101886 2024-11-16T09:58:24,013 DEBUG [RS:0;037c58a19ddb:37267-shortCompactions-0 {}] compactions.Compactor(225): Compacting f84ef8c17537456cb6d451fa35e32d14, keycount=150, bloomtype=ROW, size=13.1 K, encoding=NONE, compression=NONE, seqNum=496, earliestPutTs=1731751100079 2024-11-16T09:58:24,013 DEBUG [RS:0;037c58a19ddb:37267-longCompactions-0 {}] compactions.Compactor(225): Compacting 7ad96d41e26d439699f66cb611331ec1, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=535, earliestPutTs=1731751101924 2024-11-16T09:58:24,014 DEBUG [RS:0;037c58a19ddb:37267-shortCompactions-0 {}] compactions.Compactor(225): Compacting 2d6792fd6ed14e2abcfec73afc105145, keycount=100, bloomtype=ROW, size=9.6 K, encoding=NONE, compression=NONE, seqNum=509, earliestPutTs=1731751101886 2024-11-16T09:58:24,014 DEBUG [RS:0;037c58a19ddb:37267-longCompactions-0 {}] compactions.Compactor(225): Compacting e600d8b124904d24bf48bfa690411916, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=546, earliestPutTs=1731751103094 2024-11-16T09:58:24,014 DEBUG [RS:0;037c58a19ddb:37267-shortCompactions-0 {}] compactions.Compactor(225): Compacting 62cf785409214e10b9be44024b7bff9b, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=535, earliestPutTs=1731751101924 2024-11-16T09:58:24,015 DEBUG [RS:0;037c58a19ddb:37267-shortCompactions-0 {}] compactions.Compactor(225): Compacting 984513ede0154fe5b72bad83157b16af, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=546, earliestPutTs=1731751103094 2024-11-16T09:58:24,031 INFO [RS:0;037c58a19ddb:37267-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): d9e28756f78b0555a6733ffbf6ab98bd#A#compaction#114 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-16T09:58:24,031 DEBUG [RS:0;037c58a19ddb:37267-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/d9e28756f78b0555a6733ffbf6ab98bd/.tmp/A/b8f6f340dd554e85800945f9068c83d2 is 50, key is test_row_0/A:col10/1731751103094/Put/seqid=0 2024-11-16T09:58:24,035 INFO [RS:0;037c58a19ddb:37267-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): d9e28756f78b0555a6733ffbf6ab98bd#B#compaction#115 average throughput is 2.18 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-16T09:58:24,036 DEBUG [RS:0;037c58a19ddb:37267-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/d9e28756f78b0555a6733ffbf6ab98bd/.tmp/B/eedc3a6977eb4d788be5bede3825efa6 is 50, key is test_row_0/B:col10/1731751103094/Put/seqid=0 2024-11-16T09:58:24,046 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37591 is added to blk_1073741951_1127 (size=13595) 2024-11-16T09:58:24,052 DEBUG [RS:0;037c58a19ddb:37267-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/d9e28756f78b0555a6733ffbf6ab98bd/.tmp/A/b8f6f340dd554e85800945f9068c83d2 as hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/d9e28756f78b0555a6733ffbf6ab98bd/A/b8f6f340dd554e85800945f9068c83d2 2024-11-16T09:58:24,058 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37591 is added to blk_1073741952_1128 (size=13595) 2024-11-16T09:58:24,060 INFO [RS:0;037c58a19ddb:37267-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in d9e28756f78b0555a6733ffbf6ab98bd/A of d9e28756f78b0555a6733ffbf6ab98bd into b8f6f340dd554e85800945f9068c83d2(size=13.3 K), total size for store is 13.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-16T09:58:24,060 DEBUG [RS:0;037c58a19ddb:37267-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for d9e28756f78b0555a6733ffbf6ab98bd: 2024-11-16T09:58:24,060 INFO [RS:0;037c58a19ddb:37267-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731751082172.d9e28756f78b0555a6733ffbf6ab98bd., storeName=d9e28756f78b0555a6733ffbf6ab98bd/A, priority=12, startTime=1731751104010; duration=0sec 2024-11-16T09:58:24,060 DEBUG [RS:0;037c58a19ddb:37267-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-16T09:58:24,060 DEBUG [RS:0;037c58a19ddb:37267-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: d9e28756f78b0555a6733ffbf6ab98bd:A 2024-11-16T09:58:24,060 DEBUG [RS:0;037c58a19ddb:37267-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-16T09:58:24,061 DEBUG [RS:0;037c58a19ddb:37267-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 47918 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-16T09:58:24,062 DEBUG [RS:0;037c58a19ddb:37267-shortCompactions-0 {}] regionserver.HStore(1541): d9e28756f78b0555a6733ffbf6ab98bd/C is initiating minor compaction (all files) 2024-11-16T09:58:24,063 INFO [RS:0;037c58a19ddb:37267-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of d9e28756f78b0555a6733ffbf6ab98bd/C in TestAcidGuarantees,,1731751082172.d9e28756f78b0555a6733ffbf6ab98bd. 2024-11-16T09:58:24,063 INFO [RS:0;037c58a19ddb:37267-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/d9e28756f78b0555a6733ffbf6ab98bd/C/95c0d63049154ac794ce470e2eb2ed66, hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/d9e28756f78b0555a6733ffbf6ab98bd/C/3cfb8ef2898847a385d61a4e27bcc4b7, hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/d9e28756f78b0555a6733ffbf6ab98bd/C/6513ccc313dc49b28664372e3f14691a, hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/d9e28756f78b0555a6733ffbf6ab98bd/C/a740d4a75fb5467784531979bb9dda59] into tmpdir=hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/d9e28756f78b0555a6733ffbf6ab98bd/.tmp, totalSize=46.8 K 2024-11-16T09:58:24,063 DEBUG [RS:0;037c58a19ddb:37267-shortCompactions-0 {}] compactions.Compactor(225): Compacting 95c0d63049154ac794ce470e2eb2ed66, keycount=150, bloomtype=ROW, size=13.1 K, encoding=NONE, compression=NONE, seqNum=496, earliestPutTs=1731751100079 2024-11-16T09:58:24,064 DEBUG [RS:0;037c58a19ddb:37267-shortCompactions-0 {}] compactions.Compactor(225): Compacting 3cfb8ef2898847a385d61a4e27bcc4b7, keycount=100, bloomtype=ROW, size=9.6 K, encoding=NONE, compression=NONE, seqNum=509, earliestPutTs=1731751101886 2024-11-16T09:58:24,064 DEBUG [RS:0;037c58a19ddb:37267-shortCompactions-0 {}] compactions.Compactor(225): Compacting 6513ccc313dc49b28664372e3f14691a, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=535, earliestPutTs=1731751101924 2024-11-16T09:58:24,064 DEBUG [RS:0;037c58a19ddb:37267-shortCompactions-0 {}] compactions.Compactor(225): Compacting a740d4a75fb5467784531979bb9dda59, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=546, earliestPutTs=1731751103094 2024-11-16T09:58:24,074 INFO [RS:0;037c58a19ddb:37267-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): d9e28756f78b0555a6733ffbf6ab98bd#C#compaction#116 average throughput is unlimited, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-16T09:58:24,074 DEBUG [RS:0;037c58a19ddb:37267-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/d9e28756f78b0555a6733ffbf6ab98bd/.tmp/C/f5224569676a4236b43ae5c1ebc30ddd is 50, key is test_row_0/C:col10/1731751103094/Put/seqid=0 2024-11-16T09:58:24,082 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37591 is added to blk_1073741953_1129 (size=13595) 2024-11-16T09:58:24,093 DEBUG [RS:0;037c58a19ddb:37267-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/d9e28756f78b0555a6733ffbf6ab98bd/.tmp/C/f5224569676a4236b43ae5c1ebc30ddd as hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/d9e28756f78b0555a6733ffbf6ab98bd/C/f5224569676a4236b43ae5c1ebc30ddd 2024-11-16T09:58:24,102 INFO [RS:0;037c58a19ddb:37267-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in d9e28756f78b0555a6733ffbf6ab98bd/C of d9e28756f78b0555a6733ffbf6ab98bd into f5224569676a4236b43ae5c1ebc30ddd(size=13.3 K), total size for store is 13.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-16T09:58:24,102 DEBUG [RS:0;037c58a19ddb:37267-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for d9e28756f78b0555a6733ffbf6ab98bd: 2024-11-16T09:58:24,102 INFO [RS:0;037c58a19ddb:37267-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731751082172.d9e28756f78b0555a6733ffbf6ab98bd., storeName=d9e28756f78b0555a6733ffbf6ab98bd/C, priority=12, startTime=1731751104010; duration=0sec 2024-11-16T09:58:24,102 DEBUG [RS:0;037c58a19ddb:37267-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-16T09:58:24,102 DEBUG [RS:0;037c58a19ddb:37267-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: d9e28756f78b0555a6733ffbf6ab98bd:C 2024-11-16T09:58:24,109 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=37267 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=24 2024-11-16T09:58:24,109 DEBUG [RS_FLUSH_OPERATIONS-regionserver/037c58a19ddb:0-0 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731751082172.d9e28756f78b0555a6733ffbf6ab98bd. 2024-11-16T09:58:24,109 INFO [RS_FLUSH_OPERATIONS-regionserver/037c58a19ddb:0-0 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.HRegion(2902): Flushing d9e28756f78b0555a6733ffbf6ab98bd 3/3 column families, dataSize=147.60 KB heapSize=387.47 KB 2024-11-16T09:58:24,117 DEBUG [RS_FLUSH_OPERATIONS-regionserver/037c58a19ddb:0-0 {event_type=RS_FLUSH_REGIONS, pid=24}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/d9e28756f78b0555a6733ffbf6ab98bd/.tmp/A/1ba0292bc4774f469795331e7f47fb8b is 50, key is test_row_0/A:col10/1731751103202/Put/seqid=0 2024-11-16T09:58:24,142 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37591 is added to blk_1073741954_1130 (size=12301) 2024-11-16T09:58:24,143 INFO [RS_FLUSH_OPERATIONS-regionserver/037c58a19ddb:0-0 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=573 (bloomFilter=true), to=hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/d9e28756f78b0555a6733ffbf6ab98bd/.tmp/A/1ba0292bc4774f469795331e7f47fb8b 2024-11-16T09:58:24,156 DEBUG [RS_FLUSH_OPERATIONS-regionserver/037c58a19ddb:0-0 {event_type=RS_FLUSH_REGIONS, pid=24}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/d9e28756f78b0555a6733ffbf6ab98bd/.tmp/B/cbaff5393ba343358c22e385d872f050 is 50, key is test_row_0/B:col10/1731751103202/Put/seqid=0 2024-11-16T09:58:24,169 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37591 is added to blk_1073741955_1131 (size=12301) 2024-11-16T09:58:24,171 INFO [RS_FLUSH_OPERATIONS-regionserver/037c58a19ddb:0-0 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=573 (bloomFilter=true), to=hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/d9e28756f78b0555a6733ffbf6ab98bd/.tmp/B/cbaff5393ba343358c22e385d872f050 2024-11-16T09:58:24,183 DEBUG [RS_FLUSH_OPERATIONS-regionserver/037c58a19ddb:0-0 {event_type=RS_FLUSH_REGIONS, pid=24}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/d9e28756f78b0555a6733ffbf6ab98bd/.tmp/C/510302ea2afd46e599b5647484a585b6 is 50, key is test_row_0/C:col10/1731751103202/Put/seqid=0 2024-11-16T09:58:24,188 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37591 is added to blk_1073741956_1132 (size=12301) 2024-11-16T09:58:24,323 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37877 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=23 2024-11-16T09:58:24,355 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] regionserver.HRegion(8855): Flush requested on d9e28756f78b0555a6733ffbf6ab98bd 2024-11-16T09:58:24,355 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731751082172.d9e28756f78b0555a6733ffbf6ab98bd. as already flushing 2024-11-16T09:58:24,383 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9e28756f78b0555a6733ffbf6ab98bd, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T09:58:24,383 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] ipc.CallRunner(138): callId: 241 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39654 deadline: 1731751164378, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9e28756f78b0555a6733ffbf6ab98bd, server=037c58a19ddb,37267,1731751077844 2024-11-16T09:58:24,384 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731751082172.d9e28756f78b0555a6733ffbf6ab98bd., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 , the old value is region=TestAcidGuarantees,,1731751082172.d9e28756f78b0555a6733ffbf6ab98bd., hostname=037c58a19ddb,37267,1731751077844, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9e28756f78b0555a6733ffbf6ab98bd, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:58:24,384 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731751082172.d9e28756f78b0555a6733ffbf6ab98bd., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9e28756f78b0555a6733ffbf6ab98bd, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:58:24,384 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731751082172.d9e28756f78b0555a6733ffbf6ab98bd., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 because the exception is null or not the one we care about 2024-11-16T09:58:24,460 INFO [Thread-156 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-156. 2024-11-16T09:58:24,460 DEBUG [Thread-156 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-16T09:58:24,461 INFO [Thread-160 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-160. 2024-11-16T09:58:24,461 DEBUG [Thread-160 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-16T09:58:24,461 INFO [Thread-154 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-154. 2024-11-16T09:58:24,462 DEBUG [Thread-154 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-16T09:58:24,462 INFO [Thread-158 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-158. 2024-11-16T09:58:24,462 DEBUG [Thread-158 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-16T09:58:24,466 DEBUG [RS:0;037c58a19ddb:37267-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/d9e28756f78b0555a6733ffbf6ab98bd/.tmp/B/eedc3a6977eb4d788be5bede3825efa6 as hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/d9e28756f78b0555a6733ffbf6ab98bd/B/eedc3a6977eb4d788be5bede3825efa6 2024-11-16T09:58:24,467 DEBUG [Thread-156 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T09:58:24,467 DEBUG [Thread-154 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T09:58:24,467 DEBUG [Thread-158 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T09:58:24,467 DEBUG [Thread-160 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T09:58:24,468 DEBUG [Thread-156 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T09:58:24,468 DEBUG [Thread-158 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T09:58:24,468 DEBUG [Thread-154 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T09:58:24,468 DEBUG [Thread-160 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T09:58:24,468 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-16T09:58:24,468 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-16T09:58:24,468 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-16T09:58:24,468 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-16T09:58:24,474 INFO [RS:0;037c58a19ddb:37267-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in d9e28756f78b0555a6733ffbf6ab98bd/B of d9e28756f78b0555a6733ffbf6ab98bd into eedc3a6977eb4d788be5bede3825efa6(size=13.3 K), total size for store is 13.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-16T09:58:24,474 DEBUG [RS:0;037c58a19ddb:37267-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for d9e28756f78b0555a6733ffbf6ab98bd: 2024-11-16T09:58:24,474 INFO [RS:0;037c58a19ddb:37267-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731751082172.d9e28756f78b0555a6733ffbf6ab98bd., storeName=d9e28756f78b0555a6733ffbf6ab98bd/B, priority=12, startTime=1731751104010; duration=0sec 2024-11-16T09:58:24,474 DEBUG [RS:0;037c58a19ddb:37267-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-16T09:58:24,474 DEBUG [RS:0;037c58a19ddb:37267-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: d9e28756f78b0555a6733ffbf6ab98bd:B 2024-11-16T09:58:24,494 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9e28756f78b0555a6733ffbf6ab98bd, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T09:58:24,494 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] ipc.CallRunner(138): callId: 242 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39654 deadline: 1731751164493, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9e28756f78b0555a6733ffbf6ab98bd, server=037c58a19ddb,37267,1731751077844 2024-11-16T09:58:24,494 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731751082172.d9e28756f78b0555a6733ffbf6ab98bd., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 , the old value is region=TestAcidGuarantees,,1731751082172.d9e28756f78b0555a6733ffbf6ab98bd., hostname=037c58a19ddb,37267,1731751077844, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9e28756f78b0555a6733ffbf6ab98bd, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:58:24,494 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731751082172.d9e28756f78b0555a6733ffbf6ab98bd., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9e28756f78b0555a6733ffbf6ab98bd, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:58:24,494 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731751082172.d9e28756f78b0555a6733ffbf6ab98bd., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 because the exception is null or not the one we care about 2024-11-16T09:58:24,589 INFO [RS_FLUSH_OPERATIONS-regionserver/037c58a19ddb:0-0 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=573 (bloomFilter=true), to=hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/d9e28756f78b0555a6733ffbf6ab98bd/.tmp/C/510302ea2afd46e599b5647484a585b6 2024-11-16T09:58:24,594 DEBUG [RS_FLUSH_OPERATIONS-regionserver/037c58a19ddb:0-0 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/d9e28756f78b0555a6733ffbf6ab98bd/.tmp/A/1ba0292bc4774f469795331e7f47fb8b as hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/d9e28756f78b0555a6733ffbf6ab98bd/A/1ba0292bc4774f469795331e7f47fb8b 2024-11-16T09:58:24,598 INFO [RS_FLUSH_OPERATIONS-regionserver/037c58a19ddb:0-0 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/d9e28756f78b0555a6733ffbf6ab98bd/A/1ba0292bc4774f469795331e7f47fb8b, entries=150, sequenceid=573, filesize=12.0 K 2024-11-16T09:58:24,599 DEBUG [RS_FLUSH_OPERATIONS-regionserver/037c58a19ddb:0-0 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/d9e28756f78b0555a6733ffbf6ab98bd/.tmp/B/cbaff5393ba343358c22e385d872f050 as hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/d9e28756f78b0555a6733ffbf6ab98bd/B/cbaff5393ba343358c22e385d872f050 2024-11-16T09:58:24,603 INFO [RS_FLUSH_OPERATIONS-regionserver/037c58a19ddb:0-0 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/d9e28756f78b0555a6733ffbf6ab98bd/B/cbaff5393ba343358c22e385d872f050, entries=150, sequenceid=573, filesize=12.0 K 2024-11-16T09:58:24,604 DEBUG [RS_FLUSH_OPERATIONS-regionserver/037c58a19ddb:0-0 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/d9e28756f78b0555a6733ffbf6ab98bd/.tmp/C/510302ea2afd46e599b5647484a585b6 as hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/d9e28756f78b0555a6733ffbf6ab98bd/C/510302ea2afd46e599b5647484a585b6 2024-11-16T09:58:24,608 INFO [RS_FLUSH_OPERATIONS-regionserver/037c58a19ddb:0-0 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/d9e28756f78b0555a6733ffbf6ab98bd/C/510302ea2afd46e599b5647484a585b6, entries=150, sequenceid=573, filesize=12.0 K 2024-11-16T09:58:24,609 INFO [RS_FLUSH_OPERATIONS-regionserver/037c58a19ddb:0-0 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.HRegion(3140): Finished flush of dataSize ~147.60 KB/151140, heapSize ~387.42 KB/396720, currentSize=53.67 KB/54960 for d9e28756f78b0555a6733ffbf6ab98bd in 500ms, sequenceid=573, compaction requested=false 2024-11-16T09:58:24,609 DEBUG [RS_FLUSH_OPERATIONS-regionserver/037c58a19ddb:0-0 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.HRegion(2603): Flush status journal for d9e28756f78b0555a6733ffbf6ab98bd: 2024-11-16T09:58:24,609 DEBUG [RS_FLUSH_OPERATIONS-regionserver/037c58a19ddb:0-0 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731751082172.d9e28756f78b0555a6733ffbf6ab98bd. 2024-11-16T09:58:24,609 DEBUG [RS_FLUSH_OPERATIONS-regionserver/037c58a19ddb:0-0 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=24 2024-11-16T09:58:24,610 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37877 {}] master.HMaster(4169): Remote procedure done, pid=24 2024-11-16T09:58:24,612 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=24, resume processing ppid=23 2024-11-16T09:58:24,612 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=24, ppid=23, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.4240 sec 2024-11-16T09:58:24,614 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=23, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=23, table=TestAcidGuarantees in 1.4290 sec 2024-11-16T09:58:24,704 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] regionserver.HRegion(8855): Flush requested on d9e28756f78b0555a6733ffbf6ab98bd 2024-11-16T09:58:24,704 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-16T09:58:24,704 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing d9e28756f78b0555a6733ffbf6ab98bd 3/3 column families, dataSize=60.38 KB heapSize=158.95 KB 2024-11-16T09:58:24,705 INFO [Thread-149 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-149. 2024-11-16T09:58:24,705 DEBUG [Thread-149 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-16T09:58:24,705 DEBUG [Thread-149 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T09:58:24,705 DEBUG [Thread-149 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T09:58:24,705 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-16T09:58:24,709 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/d9e28756f78b0555a6733ffbf6ab98bd/.tmp/A/97d7157baee74526a5cc5386bac01e62 is 50, key is test_row_0/A:col10/1731751104354/Put/seqid=0 2024-11-16T09:58:24,713 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37591 is added to blk_1073741957_1133 (size=12301) 2024-11-16T09:58:25,114 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=586 (bloomFilter=true), to=hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/d9e28756f78b0555a6733ffbf6ab98bd/.tmp/A/97d7157baee74526a5cc5386bac01e62 2024-11-16T09:58:25,122 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/d9e28756f78b0555a6733ffbf6ab98bd/.tmp/B/419b545776e44644a2c05849328bb1c5 is 50, key is test_row_0/B:col10/1731751104354/Put/seqid=0 2024-11-16T09:58:25,126 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37591 is added to blk_1073741958_1134 (size=12301) 2024-11-16T09:58:25,333 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37877 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=23 2024-11-16T09:58:25,334 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-16T09:58:25,527 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=586 (bloomFilter=true), to=hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/d9e28756f78b0555a6733ffbf6ab98bd/.tmp/B/419b545776e44644a2c05849328bb1c5 2024-11-16T09:58:25,535 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/d9e28756f78b0555a6733ffbf6ab98bd/.tmp/C/fbeb8c2fb73f49599879a72b756eaeb1 is 50, key is test_row_0/C:col10/1731751104354/Put/seqid=0 2024-11-16T09:58:25,539 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37591 is added to blk_1073741959_1135 (size=12301) 2024-11-16T09:58:25,940 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=586 (bloomFilter=true), to=hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/d9e28756f78b0555a6733ffbf6ab98bd/.tmp/C/fbeb8c2fb73f49599879a72b756eaeb1 2024-11-16T09:58:25,945 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/d9e28756f78b0555a6733ffbf6ab98bd/.tmp/A/97d7157baee74526a5cc5386bac01e62 as hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/d9e28756f78b0555a6733ffbf6ab98bd/A/97d7157baee74526a5cc5386bac01e62 2024-11-16T09:58:25,949 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/d9e28756f78b0555a6733ffbf6ab98bd/A/97d7157baee74526a5cc5386bac01e62, entries=150, sequenceid=586, filesize=12.0 K 2024-11-16T09:58:25,950 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/d9e28756f78b0555a6733ffbf6ab98bd/.tmp/B/419b545776e44644a2c05849328bb1c5 as hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/d9e28756f78b0555a6733ffbf6ab98bd/B/419b545776e44644a2c05849328bb1c5 2024-11-16T09:58:25,954 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/d9e28756f78b0555a6733ffbf6ab98bd/B/419b545776e44644a2c05849328bb1c5, entries=150, sequenceid=586, filesize=12.0 K 2024-11-16T09:58:25,954 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/d9e28756f78b0555a6733ffbf6ab98bd/.tmp/C/fbeb8c2fb73f49599879a72b756eaeb1 as hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/d9e28756f78b0555a6733ffbf6ab98bd/C/fbeb8c2fb73f49599879a72b756eaeb1 2024-11-16T09:58:25,958 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/d9e28756f78b0555a6733ffbf6ab98bd/C/fbeb8c2fb73f49599879a72b756eaeb1, entries=150, sequenceid=586, filesize=12.0 K 2024-11-16T09:58:25,959 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~158.91 KB/162720, currentSize=0 B/0 for d9e28756f78b0555a6733ffbf6ab98bd in 1255ms, sequenceid=586, compaction requested=true 2024-11-16T09:58:25,959 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for d9e28756f78b0555a6733ffbf6ab98bd: 2024-11-16T09:58:25,959 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store d9e28756f78b0555a6733ffbf6ab98bd:A, priority=-2147483648, current under compaction store size is 1 2024-11-16T09:58:25,960 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-16T09:58:25,960 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store d9e28756f78b0555a6733ffbf6ab98bd:B, priority=-2147483648, current under compaction store size is 2 2024-11-16T09:58:25,960 DEBUG [RS:0;037c58a19ddb:37267-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-16T09:58:25,960 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-16T09:58:25,960 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store d9e28756f78b0555a6733ffbf6ab98bd:C, priority=-2147483648, current under compaction store size is 3 2024-11-16T09:58:25,960 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-16T09:58:25,960 DEBUG [RS:0;037c58a19ddb:37267-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-16T09:58:25,961 DEBUG [RS:0;037c58a19ddb:37267-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 38197 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-16T09:58:25,961 DEBUG [RS:0;037c58a19ddb:37267-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 38197 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-16T09:58:25,961 DEBUG [RS:0;037c58a19ddb:37267-longCompactions-0 {}] regionserver.HStore(1541): d9e28756f78b0555a6733ffbf6ab98bd/B is initiating minor compaction (all files) 2024-11-16T09:58:25,961 DEBUG [RS:0;037c58a19ddb:37267-shortCompactions-0 {}] regionserver.HStore(1541): d9e28756f78b0555a6733ffbf6ab98bd/A is initiating minor compaction (all files) 2024-11-16T09:58:25,961 INFO [RS:0;037c58a19ddb:37267-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of d9e28756f78b0555a6733ffbf6ab98bd/B in TestAcidGuarantees,,1731751082172.d9e28756f78b0555a6733ffbf6ab98bd. 2024-11-16T09:58:25,961 INFO [RS:0;037c58a19ddb:37267-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of d9e28756f78b0555a6733ffbf6ab98bd/A in TestAcidGuarantees,,1731751082172.d9e28756f78b0555a6733ffbf6ab98bd. 2024-11-16T09:58:25,961 INFO [RS:0;037c58a19ddb:37267-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/d9e28756f78b0555a6733ffbf6ab98bd/B/eedc3a6977eb4d788be5bede3825efa6, hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/d9e28756f78b0555a6733ffbf6ab98bd/B/cbaff5393ba343358c22e385d872f050, hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/d9e28756f78b0555a6733ffbf6ab98bd/B/419b545776e44644a2c05849328bb1c5] into tmpdir=hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/d9e28756f78b0555a6733ffbf6ab98bd/.tmp, totalSize=37.3 K 2024-11-16T09:58:25,961 INFO [RS:0;037c58a19ddb:37267-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/d9e28756f78b0555a6733ffbf6ab98bd/A/b8f6f340dd554e85800945f9068c83d2, hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/d9e28756f78b0555a6733ffbf6ab98bd/A/1ba0292bc4774f469795331e7f47fb8b, hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/d9e28756f78b0555a6733ffbf6ab98bd/A/97d7157baee74526a5cc5386bac01e62] into tmpdir=hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/d9e28756f78b0555a6733ffbf6ab98bd/.tmp, totalSize=37.3 K 2024-11-16T09:58:25,961 DEBUG [RS:0;037c58a19ddb:37267-longCompactions-0 {}] compactions.Compactor(225): Compacting eedc3a6977eb4d788be5bede3825efa6, keycount=150, bloomtype=ROW, size=13.3 K, encoding=NONE, compression=NONE, seqNum=546, earliestPutTs=1731751103094 2024-11-16T09:58:25,961 DEBUG [RS:0;037c58a19ddb:37267-shortCompactions-0 {}] compactions.Compactor(225): Compacting b8f6f340dd554e85800945f9068c83d2, keycount=150, bloomtype=ROW, size=13.3 K, encoding=NONE, compression=NONE, seqNum=546, earliestPutTs=1731751103094 2024-11-16T09:58:25,962 DEBUG [RS:0;037c58a19ddb:37267-longCompactions-0 {}] compactions.Compactor(225): Compacting cbaff5393ba343358c22e385d872f050, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=573, earliestPutTs=1731751103185 2024-11-16T09:58:25,962 DEBUG [RS:0;037c58a19ddb:37267-shortCompactions-0 {}] compactions.Compactor(225): Compacting 1ba0292bc4774f469795331e7f47fb8b, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=573, earliestPutTs=1731751103185 2024-11-16T09:58:25,962 DEBUG [RS:0;037c58a19ddb:37267-longCompactions-0 {}] compactions.Compactor(225): Compacting 419b545776e44644a2c05849328bb1c5, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=586, earliestPutTs=1731751104354 2024-11-16T09:58:25,962 DEBUG [RS:0;037c58a19ddb:37267-shortCompactions-0 {}] compactions.Compactor(225): Compacting 97d7157baee74526a5cc5386bac01e62, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=586, earliestPutTs=1731751104354 2024-11-16T09:58:25,971 INFO [RS:0;037c58a19ddb:37267-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): d9e28756f78b0555a6733ffbf6ab98bd#A#compaction#123 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-16T09:58:25,971 INFO [RS:0;037c58a19ddb:37267-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): d9e28756f78b0555a6733ffbf6ab98bd#B#compaction#124 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-16T09:58:25,971 DEBUG [RS:0;037c58a19ddb:37267-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/d9e28756f78b0555a6733ffbf6ab98bd/.tmp/A/5b6f13bb7dd94ef3bb875b1d9068c630 is 50, key is test_row_0/A:col10/1731751104354/Put/seqid=0 2024-11-16T09:58:25,972 DEBUG [RS:0;037c58a19ddb:37267-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/d9e28756f78b0555a6733ffbf6ab98bd/.tmp/B/f3ae6851416d4f0195116b01d3822866 is 50, key is test_row_0/B:col10/1731751104354/Put/seqid=0 2024-11-16T09:58:25,976 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37591 is added to blk_1073741960_1136 (size=13697) 2024-11-16T09:58:25,977 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37591 is added to blk_1073741961_1137 (size=13697) 2024-11-16T09:58:26,235 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtil$FsDatasetAsyncDiskServiceFixer(597): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details. 2024-11-16T09:58:26,382 DEBUG [RS:0;037c58a19ddb:37267-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/d9e28756f78b0555a6733ffbf6ab98bd/.tmp/A/5b6f13bb7dd94ef3bb875b1d9068c630 as hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/d9e28756f78b0555a6733ffbf6ab98bd/A/5b6f13bb7dd94ef3bb875b1d9068c630 2024-11-16T09:58:26,382 DEBUG [RS:0;037c58a19ddb:37267-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/d9e28756f78b0555a6733ffbf6ab98bd/.tmp/B/f3ae6851416d4f0195116b01d3822866 as hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/d9e28756f78b0555a6733ffbf6ab98bd/B/f3ae6851416d4f0195116b01d3822866 2024-11-16T09:58:26,387 INFO [RS:0;037c58a19ddb:37267-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in d9e28756f78b0555a6733ffbf6ab98bd/A of d9e28756f78b0555a6733ffbf6ab98bd into 5b6f13bb7dd94ef3bb875b1d9068c630(size=13.4 K), total size for store is 13.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-16T09:58:26,387 INFO [RS:0;037c58a19ddb:37267-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in d9e28756f78b0555a6733ffbf6ab98bd/B of d9e28756f78b0555a6733ffbf6ab98bd into f3ae6851416d4f0195116b01d3822866(size=13.4 K), total size for store is 13.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-16T09:58:26,388 DEBUG [RS:0;037c58a19ddb:37267-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for d9e28756f78b0555a6733ffbf6ab98bd: 2024-11-16T09:58:26,388 DEBUG [RS:0;037c58a19ddb:37267-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for d9e28756f78b0555a6733ffbf6ab98bd: 2024-11-16T09:58:26,388 INFO [RS:0;037c58a19ddb:37267-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731751082172.d9e28756f78b0555a6733ffbf6ab98bd., storeName=d9e28756f78b0555a6733ffbf6ab98bd/B, priority=13, startTime=1731751105960; duration=0sec 2024-11-16T09:58:26,388 INFO [RS:0;037c58a19ddb:37267-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731751082172.d9e28756f78b0555a6733ffbf6ab98bd., storeName=d9e28756f78b0555a6733ffbf6ab98bd/A, priority=13, startTime=1731751105959; duration=0sec 2024-11-16T09:58:26,388 DEBUG [RS:0;037c58a19ddb:37267-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-16T09:58:26,388 DEBUG [RS:0;037c58a19ddb:37267-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-16T09:58:26,388 DEBUG [RS:0;037c58a19ddb:37267-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: d9e28756f78b0555a6733ffbf6ab98bd:B 2024-11-16T09:58:26,388 DEBUG [RS:0;037c58a19ddb:37267-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: d9e28756f78b0555a6733ffbf6ab98bd:A 2024-11-16T09:58:26,388 DEBUG [RS:0;037c58a19ddb:37267-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-16T09:58:26,389 DEBUG [RS:0;037c58a19ddb:37267-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 38197 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-16T09:58:26,389 DEBUG [RS:0;037c58a19ddb:37267-shortCompactions-0 {}] regionserver.HStore(1541): d9e28756f78b0555a6733ffbf6ab98bd/C is initiating minor compaction (all files) 2024-11-16T09:58:26,389 INFO [RS:0;037c58a19ddb:37267-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of d9e28756f78b0555a6733ffbf6ab98bd/C in TestAcidGuarantees,,1731751082172.d9e28756f78b0555a6733ffbf6ab98bd. 2024-11-16T09:58:26,389 INFO [RS:0;037c58a19ddb:37267-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/d9e28756f78b0555a6733ffbf6ab98bd/C/f5224569676a4236b43ae5c1ebc30ddd, hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/d9e28756f78b0555a6733ffbf6ab98bd/C/510302ea2afd46e599b5647484a585b6, hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/d9e28756f78b0555a6733ffbf6ab98bd/C/fbeb8c2fb73f49599879a72b756eaeb1] into tmpdir=hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/d9e28756f78b0555a6733ffbf6ab98bd/.tmp, totalSize=37.3 K 2024-11-16T09:58:26,390 DEBUG [RS:0;037c58a19ddb:37267-shortCompactions-0 {}] compactions.Compactor(225): Compacting f5224569676a4236b43ae5c1ebc30ddd, keycount=150, bloomtype=ROW, size=13.3 K, encoding=NONE, compression=NONE, seqNum=546, earliestPutTs=1731751103094 2024-11-16T09:58:26,390 DEBUG [RS:0;037c58a19ddb:37267-shortCompactions-0 {}] compactions.Compactor(225): Compacting 510302ea2afd46e599b5647484a585b6, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=573, earliestPutTs=1731751103185 2024-11-16T09:58:26,390 DEBUG [RS:0;037c58a19ddb:37267-shortCompactions-0 {}] compactions.Compactor(225): Compacting fbeb8c2fb73f49599879a72b756eaeb1, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=586, earliestPutTs=1731751104354 2024-11-16T09:58:26,397 INFO [RS:0;037c58a19ddb:37267-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): d9e28756f78b0555a6733ffbf6ab98bd#C#compaction#125 average throughput is unlimited, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-16T09:58:26,398 DEBUG [RS:0;037c58a19ddb:37267-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/d9e28756f78b0555a6733ffbf6ab98bd/.tmp/C/df717d2a0a0149e5a5af7207c2b0a989 is 50, key is test_row_0/C:col10/1731751104354/Put/seqid=0 2024-11-16T09:58:26,401 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37591 is added to blk_1073741962_1138 (size=13697) 2024-11-16T09:58:26,654 INFO [Thread-143 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-143. 2024-11-16T09:58:26,654 DEBUG [Thread-143 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-16T09:58:26,654 DEBUG [Thread-143 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T09:58:26,655 DEBUG [Thread-143 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T09:58:26,655 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-16T09:58:26,664 INFO [Thread-147 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-147. 2024-11-16T09:58:26,664 DEBUG [Thread-147 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-16T09:58:26,664 DEBUG [Thread-147 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T09:58:26,664 DEBUG [Thread-147 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T09:58:26,664 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-16T09:58:26,674 INFO [Thread-151 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-151. 2024-11-16T09:58:26,674 DEBUG [Thread-151 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-16T09:58:26,674 DEBUG [Thread-151 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T09:58:26,674 DEBUG [Thread-151 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T09:58:26,675 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-16T09:58:26,683 INFO [Thread-145 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-145. 2024-11-16T09:58:26,683 DEBUG [Thread-145 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-16T09:58:26,683 DEBUG [Thread-145 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T09:58:26,684 DEBUG [Thread-145 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T09:58:26,684 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-16T09:58:26,684 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(392): Finished test. Writers: 2024-11-16T09:58:26,684 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 70 2024-11-16T09:58:26,684 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 75 2024-11-16T09:58:26,684 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 77 2024-11-16T09:58:26,684 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 170 2024-11-16T09:58:26,684 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 74 2024-11-16T09:58:26,684 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(396): Readers: 2024-11-16T09:58:26,684 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 3926 2024-11-16T09:58:26,684 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 4007 2024-11-16T09:58:26,684 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(400): Scanners: 2024-11-16T09:58:26,684 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1696 2024-11-16T09:58:26,684 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 5085 rows 2024-11-16T09:58:26,684 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1704 2024-11-16T09:58:26,684 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 5104 rows 2024-11-16T09:58:26,684 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-16T09:58:26,684 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool.doWork(AcidGuaranteesTestTool.java:131) at org.apache.hadoop.hbase.util.AbstractHBaseTool.run(AbstractHBaseTool.java:150) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.runTestAtomicity(AcidGuaranteesTestBase.java:102) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.runTestAtomicity(AcidGuaranteesTestBase.java:91) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.testMixedAtomicity(AcidGuaranteesTestBase.java:117) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-16T09:58:26,684 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T09:58:26,685 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T09:58:26,685 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-16T09:58:26,689 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37877 {}] master.HMaster$13(2820): Client=jenkins//172.17.0.2 disable TestAcidGuarantees 2024-11-16T09:58:26,693 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37877 {}] procedure2.ProcedureExecutor(1139): Stored pid=25, state=RUNNABLE:DISABLE_TABLE_PREPARE, hasLock=false; DisableTableProcedure table=TestAcidGuarantees 2024-11-16T09:58:26,695 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37877 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=25 2024-11-16T09:58:26,695 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731751106695"}]},"ts":"1731751106695"} 2024-11-16T09:58:26,697 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLING in hbase:meta 2024-11-16T09:58:26,697 INFO [PEWorker-1 {}] procedure.DisableTableProcedure(284): Set TestAcidGuarantees to state=DISABLING 2024-11-16T09:58:26,699 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=26, ppid=25, state=RUNNABLE:CLOSE_TABLE_REGIONS_SCHEDULE, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees}] 2024-11-16T09:58:26,703 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=27, ppid=26, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=d9e28756f78b0555a6733ffbf6ab98bd, UNASSIGN}] 2024-11-16T09:58:26,703 INFO [PEWorker-2 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=27, ppid=26, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=d9e28756f78b0555a6733ffbf6ab98bd, UNASSIGN 2024-11-16T09:58:26,705 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=27 updating hbase:meta row=d9e28756f78b0555a6733ffbf6ab98bd, regionState=CLOSING, regionLocation=037c58a19ddb,37267,1731751077844 2024-11-16T09:58:26,707 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=27, ppid=26, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=d9e28756f78b0555a6733ffbf6ab98bd, UNASSIGN because future has completed 2024-11-16T09:58:26,707 DEBUG [PEWorker-5 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-11-16T09:58:26,707 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=28, ppid=27, state=RUNNABLE, hasLock=false; CloseRegionProcedure d9e28756f78b0555a6733ffbf6ab98bd, server=037c58a19ddb,37267,1731751077844}] 2024-11-16T09:58:26,802 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37877 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=25 2024-11-16T09:58:26,808 DEBUG [RS:0;037c58a19ddb:37267-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/d9e28756f78b0555a6733ffbf6ab98bd/.tmp/C/df717d2a0a0149e5a5af7207c2b0a989 as hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/d9e28756f78b0555a6733ffbf6ab98bd/C/df717d2a0a0149e5a5af7207c2b0a989 2024-11-16T09:58:26,813 INFO [RS:0;037c58a19ddb:37267-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in d9e28756f78b0555a6733ffbf6ab98bd/C of d9e28756f78b0555a6733ffbf6ab98bd into df717d2a0a0149e5a5af7207c2b0a989(size=13.4 K), total size for store is 13.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-16T09:58:26,813 DEBUG [RS:0;037c58a19ddb:37267-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for d9e28756f78b0555a6733ffbf6ab98bd: 2024-11-16T09:58:26,813 INFO [RS:0;037c58a19ddb:37267-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731751082172.d9e28756f78b0555a6733ffbf6ab98bd., storeName=d9e28756f78b0555a6733ffbf6ab98bd/C, priority=13, startTime=1731751105960; duration=0sec 2024-11-16T09:58:26,813 DEBUG [RS:0;037c58a19ddb:37267-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-16T09:58:26,813 DEBUG [RS:0;037c58a19ddb:37267-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: d9e28756f78b0555a6733ffbf6ab98bd:C 2024-11-16T09:58:26,863 INFO [RS_CLOSE_REGION-regionserver/037c58a19ddb:0-0 {event_type=M_RS_CLOSE_REGION, pid=28}] handler.UnassignRegionHandler(122): Close d9e28756f78b0555a6733ffbf6ab98bd 2024-11-16T09:58:26,864 DEBUG [RS_CLOSE_REGION-regionserver/037c58a19ddb:0-0 {event_type=M_RS_CLOSE_REGION, pid=28}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-11-16T09:58:26,864 DEBUG [RS_CLOSE_REGION-regionserver/037c58a19ddb:0-0 {event_type=M_RS_CLOSE_REGION, pid=28}] regionserver.HRegion(1722): Closing d9e28756f78b0555a6733ffbf6ab98bd, disabling compactions & flushes 2024-11-16T09:58:26,864 INFO [RS_CLOSE_REGION-regionserver/037c58a19ddb:0-0 {event_type=M_RS_CLOSE_REGION, pid=28}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1731751082172.d9e28756f78b0555a6733ffbf6ab98bd. 2024-11-16T09:58:26,864 DEBUG [RS_CLOSE_REGION-regionserver/037c58a19ddb:0-0 {event_type=M_RS_CLOSE_REGION, pid=28}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1731751082172.d9e28756f78b0555a6733ffbf6ab98bd. 2024-11-16T09:58:26,864 DEBUG [RS_CLOSE_REGION-regionserver/037c58a19ddb:0-0 {event_type=M_RS_CLOSE_REGION, pid=28}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1731751082172.d9e28756f78b0555a6733ffbf6ab98bd. after waiting 0 ms 2024-11-16T09:58:26,864 DEBUG [RS_CLOSE_REGION-regionserver/037c58a19ddb:0-0 {event_type=M_RS_CLOSE_REGION, pid=28}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1731751082172.d9e28756f78b0555a6733ffbf6ab98bd. 2024-11-16T09:58:26,865 INFO [RS_CLOSE_REGION-regionserver/037c58a19ddb:0-0 {event_type=M_RS_CLOSE_REGION, pid=28}] regionserver.HRegion(2902): Flushing d9e28756f78b0555a6733ffbf6ab98bd 3/3 column families, dataSize=26.84 KB heapSize=71.06 KB 2024-11-16T09:58:26,869 DEBUG [RS_CLOSE_REGION-regionserver/037c58a19ddb:0-0 {event_type=M_RS_CLOSE_REGION, pid=28}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/d9e28756f78b0555a6733ffbf6ab98bd/.tmp/A/a9b2865d03dd45cd95d4c39c597e5f93 is 50, key is test_row_0/A:col10/1731751106682/Put/seqid=0 2024-11-16T09:58:26,872 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37591 is added to blk_1073741963_1139 (size=12301) 2024-11-16T09:58:27,013 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37877 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=25 2024-11-16T09:58:27,274 INFO [RS_CLOSE_REGION-regionserver/037c58a19ddb:0-0 {event_type=M_RS_CLOSE_REGION, pid=28}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=8.95 KB at sequenceid=596 (bloomFilter=true), to=hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/d9e28756f78b0555a6733ffbf6ab98bd/.tmp/A/a9b2865d03dd45cd95d4c39c597e5f93 2024-11-16T09:58:27,283 DEBUG [RS_CLOSE_REGION-regionserver/037c58a19ddb:0-0 {event_type=M_RS_CLOSE_REGION, pid=28}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/d9e28756f78b0555a6733ffbf6ab98bd/.tmp/B/f10c3c606923418e9ae4d3c253f985b5 is 50, key is test_row_0/B:col10/1731751106682/Put/seqid=0 2024-11-16T09:58:27,286 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37591 is added to blk_1073741964_1140 (size=12301) 2024-11-16T09:58:27,323 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37877 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=25 2024-11-16T09:58:27,687 INFO [RS_CLOSE_REGION-regionserver/037c58a19ddb:0-0 {event_type=M_RS_CLOSE_REGION, pid=28}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=8.95 KB at sequenceid=596 (bloomFilter=true), to=hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/d9e28756f78b0555a6733ffbf6ab98bd/.tmp/B/f10c3c606923418e9ae4d3c253f985b5 2024-11-16T09:58:27,695 DEBUG [RS_CLOSE_REGION-regionserver/037c58a19ddb:0-0 {event_type=M_RS_CLOSE_REGION, pid=28}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/d9e28756f78b0555a6733ffbf6ab98bd/.tmp/C/9f5f3912f5d94654847f8f9a2d3bc588 is 50, key is test_row_0/C:col10/1731751106682/Put/seqid=0 2024-11-16T09:58:27,698 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37591 is added to blk_1073741965_1141 (size=12301) 2024-11-16T09:58:27,833 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37877 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=25 2024-11-16T09:58:28,099 INFO [RS_CLOSE_REGION-regionserver/037c58a19ddb:0-0 {event_type=M_RS_CLOSE_REGION, pid=28}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=8.95 KB at sequenceid=596 (bloomFilter=true), to=hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/d9e28756f78b0555a6733ffbf6ab98bd/.tmp/C/9f5f3912f5d94654847f8f9a2d3bc588 2024-11-16T09:58:28,105 DEBUG [RS_CLOSE_REGION-regionserver/037c58a19ddb:0-0 {event_type=M_RS_CLOSE_REGION, pid=28}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/d9e28756f78b0555a6733ffbf6ab98bd/.tmp/A/a9b2865d03dd45cd95d4c39c597e5f93 as hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/d9e28756f78b0555a6733ffbf6ab98bd/A/a9b2865d03dd45cd95d4c39c597e5f93 2024-11-16T09:58:28,110 INFO [RS_CLOSE_REGION-regionserver/037c58a19ddb:0-0 {event_type=M_RS_CLOSE_REGION, pid=28}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/d9e28756f78b0555a6733ffbf6ab98bd/A/a9b2865d03dd45cd95d4c39c597e5f93, entries=150, sequenceid=596, filesize=12.0 K 2024-11-16T09:58:28,111 DEBUG [RS_CLOSE_REGION-regionserver/037c58a19ddb:0-0 {event_type=M_RS_CLOSE_REGION, pid=28}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/d9e28756f78b0555a6733ffbf6ab98bd/.tmp/B/f10c3c606923418e9ae4d3c253f985b5 as hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/d9e28756f78b0555a6733ffbf6ab98bd/B/f10c3c606923418e9ae4d3c253f985b5 2024-11-16T09:58:28,115 INFO [RS_CLOSE_REGION-regionserver/037c58a19ddb:0-0 {event_type=M_RS_CLOSE_REGION, pid=28}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/d9e28756f78b0555a6733ffbf6ab98bd/B/f10c3c606923418e9ae4d3c253f985b5, entries=150, sequenceid=596, filesize=12.0 K 2024-11-16T09:58:28,116 DEBUG [RS_CLOSE_REGION-regionserver/037c58a19ddb:0-0 {event_type=M_RS_CLOSE_REGION, pid=28}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/d9e28756f78b0555a6733ffbf6ab98bd/.tmp/C/9f5f3912f5d94654847f8f9a2d3bc588 as hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/d9e28756f78b0555a6733ffbf6ab98bd/C/9f5f3912f5d94654847f8f9a2d3bc588 2024-11-16T09:58:28,120 INFO [RS_CLOSE_REGION-regionserver/037c58a19ddb:0-0 {event_type=M_RS_CLOSE_REGION, pid=28}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/d9e28756f78b0555a6733ffbf6ab98bd/C/9f5f3912f5d94654847f8f9a2d3bc588, entries=150, sequenceid=596, filesize=12.0 K 2024-11-16T09:58:28,120 INFO [RS_CLOSE_REGION-regionserver/037c58a19ddb:0-0 {event_type=M_RS_CLOSE_REGION, pid=28}] regionserver.HRegion(3140): Finished flush of dataSize ~26.84 KB/27480, heapSize ~71.02 KB/72720, currentSize=0 B/0 for d9e28756f78b0555a6733ffbf6ab98bd in 1256ms, sequenceid=596, compaction requested=false 2024-11-16T09:58:28,121 DEBUG [StoreCloser-TestAcidGuarantees,,1731751082172.d9e28756f78b0555a6733ffbf6ab98bd.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/d9e28756f78b0555a6733ffbf6ab98bd/A/08ff50e864824ad0a3fd6cbd2d93846b, hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/d9e28756f78b0555a6733ffbf6ab98bd/A/263ffc1427114090a4723fb2b1539548, hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/d9e28756f78b0555a6733ffbf6ab98bd/A/8f9655c1866245c9a20f69a472c4ca2b, hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/d9e28756f78b0555a6733ffbf6ab98bd/A/fd8aea0564294c4d927efa1b14d23126, hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/d9e28756f78b0555a6733ffbf6ab98bd/A/709c635778e04296bc2bbe6d259b56a0, hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/d9e28756f78b0555a6733ffbf6ab98bd/A/5bdf9040556249c8890d8df9eff5b42c, hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/d9e28756f78b0555a6733ffbf6ab98bd/A/6089e65fdbf4499499621e82097c69ab, hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/d9e28756f78b0555a6733ffbf6ab98bd/A/ef206033b0624c7a94ea8929d5fca761, hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/d9e28756f78b0555a6733ffbf6ab98bd/A/43a771f3ee484bb4936eea002ad0ec6c, hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/d9e28756f78b0555a6733ffbf6ab98bd/A/a919ab9e3a914bc59f13fe1f3f7b251e, hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/d9e28756f78b0555a6733ffbf6ab98bd/A/8900f6b9a93d4df2b2b02e97cb83983f, hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/d9e28756f78b0555a6733ffbf6ab98bd/A/5cd12febed204efabe434b45e92eccae, hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/d9e28756f78b0555a6733ffbf6ab98bd/A/0dbb7de1f967420a83f7452d01da47f7, hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/d9e28756f78b0555a6733ffbf6ab98bd/A/d1a50bbda2704aeead00924112cdb2ed, hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/d9e28756f78b0555a6733ffbf6ab98bd/A/b68a7ab34a8847e5a7139a4a8ec2b414, hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/d9e28756f78b0555a6733ffbf6ab98bd/A/2108968e9c084ac5bdc90f8e037e0627, hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/d9e28756f78b0555a6733ffbf6ab98bd/A/f18628f693944e64aadc099e58410220, hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/d9e28756f78b0555a6733ffbf6ab98bd/A/24edc08c1cf04c16bf00fef927979300, hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/d9e28756f78b0555a6733ffbf6ab98bd/A/4b3481cf62e34e3cbb47eb7e169c35c5, hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/d9e28756f78b0555a6733ffbf6ab98bd/A/d737eb9df45049199ca0e28c20466aed, hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/d9e28756f78b0555a6733ffbf6ab98bd/A/3a30ee055c414e25a9cecba7e9183dbe, hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/d9e28756f78b0555a6733ffbf6ab98bd/A/f2ad538a347b4ca9bf7afd4e658e5bac, hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/d9e28756f78b0555a6733ffbf6ab98bd/A/18344b8f21c345efa4b177f11720008e, hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/d9e28756f78b0555a6733ffbf6ab98bd/A/32643ed1b9394c95a86ca96dcf7364dd, hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/d9e28756f78b0555a6733ffbf6ab98bd/A/d1f88cc47aef44b2a64aed41364c5430, hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/d9e28756f78b0555a6733ffbf6ab98bd/A/193f043412fd40d3989d5e43a5329301, hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/d9e28756f78b0555a6733ffbf6ab98bd/A/2eb7658b24074f5f91f101c2bef2e6c7, hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/d9e28756f78b0555a6733ffbf6ab98bd/A/98ef79933cf949c4aaba70bd1f16827e, hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/d9e28756f78b0555a6733ffbf6ab98bd/A/4bbd180dd1ad4a2884e953f2b35b43be, hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/d9e28756f78b0555a6733ffbf6ab98bd/A/5218094697594d79beed890deb9c28c5, hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/d9e28756f78b0555a6733ffbf6ab98bd/A/6e28e90f94ae4c7cb291c53768fd3408, hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/d9e28756f78b0555a6733ffbf6ab98bd/A/6ba55af74e864857b6100bf2a5217c3d, hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/d9e28756f78b0555a6733ffbf6ab98bd/A/4c0a53ff6af042a7ab43c7f78203daec, hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/d9e28756f78b0555a6733ffbf6ab98bd/A/f84ef8c17537456cb6d451fa35e32d14, hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/d9e28756f78b0555a6733ffbf6ab98bd/A/92a11abe45c74a93a787b68e82adc154, hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/d9e28756f78b0555a6733ffbf6ab98bd/A/2d6792fd6ed14e2abcfec73afc105145, hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/d9e28756f78b0555a6733ffbf6ab98bd/A/62cf785409214e10b9be44024b7bff9b, hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/d9e28756f78b0555a6733ffbf6ab98bd/A/984513ede0154fe5b72bad83157b16af, hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/d9e28756f78b0555a6733ffbf6ab98bd/A/b8f6f340dd554e85800945f9068c83d2, hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/d9e28756f78b0555a6733ffbf6ab98bd/A/1ba0292bc4774f469795331e7f47fb8b, hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/d9e28756f78b0555a6733ffbf6ab98bd/A/97d7157baee74526a5cc5386bac01e62] to archive 2024-11-16T09:58:28,124 DEBUG [StoreCloser-TestAcidGuarantees,,1731751082172.d9e28756f78b0555a6733ffbf6ab98bd.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-16T09:58:28,129 DEBUG [StoreCloser-TestAcidGuarantees,,1731751082172.d9e28756f78b0555a6733ffbf6ab98bd.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/d9e28756f78b0555a6733ffbf6ab98bd/A/08ff50e864824ad0a3fd6cbd2d93846b to hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/archive/data/default/TestAcidGuarantees/d9e28756f78b0555a6733ffbf6ab98bd/A/08ff50e864824ad0a3fd6cbd2d93846b 2024-11-16T09:58:28,131 DEBUG [StoreCloser-TestAcidGuarantees,,1731751082172.d9e28756f78b0555a6733ffbf6ab98bd.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/d9e28756f78b0555a6733ffbf6ab98bd/A/263ffc1427114090a4723fb2b1539548 to hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/archive/data/default/TestAcidGuarantees/d9e28756f78b0555a6733ffbf6ab98bd/A/263ffc1427114090a4723fb2b1539548 2024-11-16T09:58:28,132 DEBUG [StoreCloser-TestAcidGuarantees,,1731751082172.d9e28756f78b0555a6733ffbf6ab98bd.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/d9e28756f78b0555a6733ffbf6ab98bd/A/8f9655c1866245c9a20f69a472c4ca2b to hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/archive/data/default/TestAcidGuarantees/d9e28756f78b0555a6733ffbf6ab98bd/A/8f9655c1866245c9a20f69a472c4ca2b 2024-11-16T09:58:28,133 DEBUG [StoreCloser-TestAcidGuarantees,,1731751082172.d9e28756f78b0555a6733ffbf6ab98bd.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/d9e28756f78b0555a6733ffbf6ab98bd/A/fd8aea0564294c4d927efa1b14d23126 to hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/archive/data/default/TestAcidGuarantees/d9e28756f78b0555a6733ffbf6ab98bd/A/fd8aea0564294c4d927efa1b14d23126 2024-11-16T09:58:28,134 DEBUG [StoreCloser-TestAcidGuarantees,,1731751082172.d9e28756f78b0555a6733ffbf6ab98bd.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/d9e28756f78b0555a6733ffbf6ab98bd/A/709c635778e04296bc2bbe6d259b56a0 to hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/archive/data/default/TestAcidGuarantees/d9e28756f78b0555a6733ffbf6ab98bd/A/709c635778e04296bc2bbe6d259b56a0 2024-11-16T09:58:28,136 DEBUG [StoreCloser-TestAcidGuarantees,,1731751082172.d9e28756f78b0555a6733ffbf6ab98bd.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/d9e28756f78b0555a6733ffbf6ab98bd/A/5bdf9040556249c8890d8df9eff5b42c to hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/archive/data/default/TestAcidGuarantees/d9e28756f78b0555a6733ffbf6ab98bd/A/5bdf9040556249c8890d8df9eff5b42c 2024-11-16T09:58:28,137 DEBUG [StoreCloser-TestAcidGuarantees,,1731751082172.d9e28756f78b0555a6733ffbf6ab98bd.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/d9e28756f78b0555a6733ffbf6ab98bd/A/6089e65fdbf4499499621e82097c69ab to hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/archive/data/default/TestAcidGuarantees/d9e28756f78b0555a6733ffbf6ab98bd/A/6089e65fdbf4499499621e82097c69ab 2024-11-16T09:58:28,138 DEBUG [StoreCloser-TestAcidGuarantees,,1731751082172.d9e28756f78b0555a6733ffbf6ab98bd.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/d9e28756f78b0555a6733ffbf6ab98bd/A/ef206033b0624c7a94ea8929d5fca761 to hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/archive/data/default/TestAcidGuarantees/d9e28756f78b0555a6733ffbf6ab98bd/A/ef206033b0624c7a94ea8929d5fca761 2024-11-16T09:58:28,139 DEBUG [StoreCloser-TestAcidGuarantees,,1731751082172.d9e28756f78b0555a6733ffbf6ab98bd.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/d9e28756f78b0555a6733ffbf6ab98bd/A/43a771f3ee484bb4936eea002ad0ec6c to hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/archive/data/default/TestAcidGuarantees/d9e28756f78b0555a6733ffbf6ab98bd/A/43a771f3ee484bb4936eea002ad0ec6c 2024-11-16T09:58:28,140 DEBUG [StoreCloser-TestAcidGuarantees,,1731751082172.d9e28756f78b0555a6733ffbf6ab98bd.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/d9e28756f78b0555a6733ffbf6ab98bd/A/a919ab9e3a914bc59f13fe1f3f7b251e to hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/archive/data/default/TestAcidGuarantees/d9e28756f78b0555a6733ffbf6ab98bd/A/a919ab9e3a914bc59f13fe1f3f7b251e 2024-11-16T09:58:28,142 DEBUG [StoreCloser-TestAcidGuarantees,,1731751082172.d9e28756f78b0555a6733ffbf6ab98bd.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/d9e28756f78b0555a6733ffbf6ab98bd/A/8900f6b9a93d4df2b2b02e97cb83983f to hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/archive/data/default/TestAcidGuarantees/d9e28756f78b0555a6733ffbf6ab98bd/A/8900f6b9a93d4df2b2b02e97cb83983f 2024-11-16T09:58:28,143 DEBUG [StoreCloser-TestAcidGuarantees,,1731751082172.d9e28756f78b0555a6733ffbf6ab98bd.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/d9e28756f78b0555a6733ffbf6ab98bd/A/5cd12febed204efabe434b45e92eccae to hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/archive/data/default/TestAcidGuarantees/d9e28756f78b0555a6733ffbf6ab98bd/A/5cd12febed204efabe434b45e92eccae 2024-11-16T09:58:28,144 DEBUG [StoreCloser-TestAcidGuarantees,,1731751082172.d9e28756f78b0555a6733ffbf6ab98bd.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/d9e28756f78b0555a6733ffbf6ab98bd/A/0dbb7de1f967420a83f7452d01da47f7 to hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/archive/data/default/TestAcidGuarantees/d9e28756f78b0555a6733ffbf6ab98bd/A/0dbb7de1f967420a83f7452d01da47f7 2024-11-16T09:58:28,145 DEBUG [StoreCloser-TestAcidGuarantees,,1731751082172.d9e28756f78b0555a6733ffbf6ab98bd.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/d9e28756f78b0555a6733ffbf6ab98bd/A/d1a50bbda2704aeead00924112cdb2ed to hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/archive/data/default/TestAcidGuarantees/d9e28756f78b0555a6733ffbf6ab98bd/A/d1a50bbda2704aeead00924112cdb2ed 2024-11-16T09:58:28,146 DEBUG [StoreCloser-TestAcidGuarantees,,1731751082172.d9e28756f78b0555a6733ffbf6ab98bd.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/d9e28756f78b0555a6733ffbf6ab98bd/A/b68a7ab34a8847e5a7139a4a8ec2b414 to hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/archive/data/default/TestAcidGuarantees/d9e28756f78b0555a6733ffbf6ab98bd/A/b68a7ab34a8847e5a7139a4a8ec2b414 2024-11-16T09:58:28,148 DEBUG [StoreCloser-TestAcidGuarantees,,1731751082172.d9e28756f78b0555a6733ffbf6ab98bd.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/d9e28756f78b0555a6733ffbf6ab98bd/A/2108968e9c084ac5bdc90f8e037e0627 to hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/archive/data/default/TestAcidGuarantees/d9e28756f78b0555a6733ffbf6ab98bd/A/2108968e9c084ac5bdc90f8e037e0627 2024-11-16T09:58:28,149 DEBUG [StoreCloser-TestAcidGuarantees,,1731751082172.d9e28756f78b0555a6733ffbf6ab98bd.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/d9e28756f78b0555a6733ffbf6ab98bd/A/f18628f693944e64aadc099e58410220 to hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/archive/data/default/TestAcidGuarantees/d9e28756f78b0555a6733ffbf6ab98bd/A/f18628f693944e64aadc099e58410220 2024-11-16T09:58:28,150 DEBUG [StoreCloser-TestAcidGuarantees,,1731751082172.d9e28756f78b0555a6733ffbf6ab98bd.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/d9e28756f78b0555a6733ffbf6ab98bd/A/24edc08c1cf04c16bf00fef927979300 to hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/archive/data/default/TestAcidGuarantees/d9e28756f78b0555a6733ffbf6ab98bd/A/24edc08c1cf04c16bf00fef927979300 2024-11-16T09:58:28,152 DEBUG [StoreCloser-TestAcidGuarantees,,1731751082172.d9e28756f78b0555a6733ffbf6ab98bd.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/d9e28756f78b0555a6733ffbf6ab98bd/A/4b3481cf62e34e3cbb47eb7e169c35c5 to hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/archive/data/default/TestAcidGuarantees/d9e28756f78b0555a6733ffbf6ab98bd/A/4b3481cf62e34e3cbb47eb7e169c35c5 2024-11-16T09:58:28,153 DEBUG [StoreCloser-TestAcidGuarantees,,1731751082172.d9e28756f78b0555a6733ffbf6ab98bd.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/d9e28756f78b0555a6733ffbf6ab98bd/A/d737eb9df45049199ca0e28c20466aed to hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/archive/data/default/TestAcidGuarantees/d9e28756f78b0555a6733ffbf6ab98bd/A/d737eb9df45049199ca0e28c20466aed 2024-11-16T09:58:28,154 DEBUG [StoreCloser-TestAcidGuarantees,,1731751082172.d9e28756f78b0555a6733ffbf6ab98bd.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/d9e28756f78b0555a6733ffbf6ab98bd/A/3a30ee055c414e25a9cecba7e9183dbe to hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/archive/data/default/TestAcidGuarantees/d9e28756f78b0555a6733ffbf6ab98bd/A/3a30ee055c414e25a9cecba7e9183dbe 2024-11-16T09:58:28,156 DEBUG [StoreCloser-TestAcidGuarantees,,1731751082172.d9e28756f78b0555a6733ffbf6ab98bd.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/d9e28756f78b0555a6733ffbf6ab98bd/A/f2ad538a347b4ca9bf7afd4e658e5bac to hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/archive/data/default/TestAcidGuarantees/d9e28756f78b0555a6733ffbf6ab98bd/A/f2ad538a347b4ca9bf7afd4e658e5bac 2024-11-16T09:58:28,157 DEBUG [StoreCloser-TestAcidGuarantees,,1731751082172.d9e28756f78b0555a6733ffbf6ab98bd.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/d9e28756f78b0555a6733ffbf6ab98bd/A/18344b8f21c345efa4b177f11720008e to hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/archive/data/default/TestAcidGuarantees/d9e28756f78b0555a6733ffbf6ab98bd/A/18344b8f21c345efa4b177f11720008e 2024-11-16T09:58:28,158 DEBUG [StoreCloser-TestAcidGuarantees,,1731751082172.d9e28756f78b0555a6733ffbf6ab98bd.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/d9e28756f78b0555a6733ffbf6ab98bd/A/32643ed1b9394c95a86ca96dcf7364dd to hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/archive/data/default/TestAcidGuarantees/d9e28756f78b0555a6733ffbf6ab98bd/A/32643ed1b9394c95a86ca96dcf7364dd 2024-11-16T09:58:28,159 DEBUG [StoreCloser-TestAcidGuarantees,,1731751082172.d9e28756f78b0555a6733ffbf6ab98bd.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/d9e28756f78b0555a6733ffbf6ab98bd/A/d1f88cc47aef44b2a64aed41364c5430 to hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/archive/data/default/TestAcidGuarantees/d9e28756f78b0555a6733ffbf6ab98bd/A/d1f88cc47aef44b2a64aed41364c5430 2024-11-16T09:58:28,161 DEBUG [StoreCloser-TestAcidGuarantees,,1731751082172.d9e28756f78b0555a6733ffbf6ab98bd.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/d9e28756f78b0555a6733ffbf6ab98bd/A/193f043412fd40d3989d5e43a5329301 to hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/archive/data/default/TestAcidGuarantees/d9e28756f78b0555a6733ffbf6ab98bd/A/193f043412fd40d3989d5e43a5329301 2024-11-16T09:58:28,162 DEBUG [StoreCloser-TestAcidGuarantees,,1731751082172.d9e28756f78b0555a6733ffbf6ab98bd.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/d9e28756f78b0555a6733ffbf6ab98bd/A/2eb7658b24074f5f91f101c2bef2e6c7 to hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/archive/data/default/TestAcidGuarantees/d9e28756f78b0555a6733ffbf6ab98bd/A/2eb7658b24074f5f91f101c2bef2e6c7 2024-11-16T09:58:28,163 DEBUG [StoreCloser-TestAcidGuarantees,,1731751082172.d9e28756f78b0555a6733ffbf6ab98bd.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/d9e28756f78b0555a6733ffbf6ab98bd/A/98ef79933cf949c4aaba70bd1f16827e to hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/archive/data/default/TestAcidGuarantees/d9e28756f78b0555a6733ffbf6ab98bd/A/98ef79933cf949c4aaba70bd1f16827e 2024-11-16T09:58:28,164 DEBUG [StoreCloser-TestAcidGuarantees,,1731751082172.d9e28756f78b0555a6733ffbf6ab98bd.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/d9e28756f78b0555a6733ffbf6ab98bd/A/4bbd180dd1ad4a2884e953f2b35b43be to hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/archive/data/default/TestAcidGuarantees/d9e28756f78b0555a6733ffbf6ab98bd/A/4bbd180dd1ad4a2884e953f2b35b43be 2024-11-16T09:58:28,166 DEBUG [StoreCloser-TestAcidGuarantees,,1731751082172.d9e28756f78b0555a6733ffbf6ab98bd.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/d9e28756f78b0555a6733ffbf6ab98bd/A/5218094697594d79beed890deb9c28c5 to hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/archive/data/default/TestAcidGuarantees/d9e28756f78b0555a6733ffbf6ab98bd/A/5218094697594d79beed890deb9c28c5 2024-11-16T09:58:28,167 DEBUG [StoreCloser-TestAcidGuarantees,,1731751082172.d9e28756f78b0555a6733ffbf6ab98bd.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/d9e28756f78b0555a6733ffbf6ab98bd/A/6e28e90f94ae4c7cb291c53768fd3408 to hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/archive/data/default/TestAcidGuarantees/d9e28756f78b0555a6733ffbf6ab98bd/A/6e28e90f94ae4c7cb291c53768fd3408 2024-11-16T09:58:28,168 DEBUG [StoreCloser-TestAcidGuarantees,,1731751082172.d9e28756f78b0555a6733ffbf6ab98bd.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/d9e28756f78b0555a6733ffbf6ab98bd/A/6ba55af74e864857b6100bf2a5217c3d to hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/archive/data/default/TestAcidGuarantees/d9e28756f78b0555a6733ffbf6ab98bd/A/6ba55af74e864857b6100bf2a5217c3d 2024-11-16T09:58:28,169 DEBUG [StoreCloser-TestAcidGuarantees,,1731751082172.d9e28756f78b0555a6733ffbf6ab98bd.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/d9e28756f78b0555a6733ffbf6ab98bd/A/4c0a53ff6af042a7ab43c7f78203daec to hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/archive/data/default/TestAcidGuarantees/d9e28756f78b0555a6733ffbf6ab98bd/A/4c0a53ff6af042a7ab43c7f78203daec 2024-11-16T09:58:28,170 DEBUG [StoreCloser-TestAcidGuarantees,,1731751082172.d9e28756f78b0555a6733ffbf6ab98bd.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/d9e28756f78b0555a6733ffbf6ab98bd/A/f84ef8c17537456cb6d451fa35e32d14 to hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/archive/data/default/TestAcidGuarantees/d9e28756f78b0555a6733ffbf6ab98bd/A/f84ef8c17537456cb6d451fa35e32d14 2024-11-16T09:58:28,172 DEBUG [StoreCloser-TestAcidGuarantees,,1731751082172.d9e28756f78b0555a6733ffbf6ab98bd.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/d9e28756f78b0555a6733ffbf6ab98bd/A/92a11abe45c74a93a787b68e82adc154 to hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/archive/data/default/TestAcidGuarantees/d9e28756f78b0555a6733ffbf6ab98bd/A/92a11abe45c74a93a787b68e82adc154 2024-11-16T09:58:28,173 DEBUG [StoreCloser-TestAcidGuarantees,,1731751082172.d9e28756f78b0555a6733ffbf6ab98bd.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/d9e28756f78b0555a6733ffbf6ab98bd/A/2d6792fd6ed14e2abcfec73afc105145 to hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/archive/data/default/TestAcidGuarantees/d9e28756f78b0555a6733ffbf6ab98bd/A/2d6792fd6ed14e2abcfec73afc105145 2024-11-16T09:58:28,174 DEBUG [StoreCloser-TestAcidGuarantees,,1731751082172.d9e28756f78b0555a6733ffbf6ab98bd.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/d9e28756f78b0555a6733ffbf6ab98bd/A/62cf785409214e10b9be44024b7bff9b to hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/archive/data/default/TestAcidGuarantees/d9e28756f78b0555a6733ffbf6ab98bd/A/62cf785409214e10b9be44024b7bff9b 2024-11-16T09:58:28,175 DEBUG [StoreCloser-TestAcidGuarantees,,1731751082172.d9e28756f78b0555a6733ffbf6ab98bd.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/d9e28756f78b0555a6733ffbf6ab98bd/A/984513ede0154fe5b72bad83157b16af to hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/archive/data/default/TestAcidGuarantees/d9e28756f78b0555a6733ffbf6ab98bd/A/984513ede0154fe5b72bad83157b16af 2024-11-16T09:58:28,177 DEBUG [StoreCloser-TestAcidGuarantees,,1731751082172.d9e28756f78b0555a6733ffbf6ab98bd.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/d9e28756f78b0555a6733ffbf6ab98bd/A/b8f6f340dd554e85800945f9068c83d2 to hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/archive/data/default/TestAcidGuarantees/d9e28756f78b0555a6733ffbf6ab98bd/A/b8f6f340dd554e85800945f9068c83d2 2024-11-16T09:58:28,178 DEBUG [StoreCloser-TestAcidGuarantees,,1731751082172.d9e28756f78b0555a6733ffbf6ab98bd.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/d9e28756f78b0555a6733ffbf6ab98bd/A/1ba0292bc4774f469795331e7f47fb8b to hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/archive/data/default/TestAcidGuarantees/d9e28756f78b0555a6733ffbf6ab98bd/A/1ba0292bc4774f469795331e7f47fb8b 2024-11-16T09:58:28,179 DEBUG [StoreCloser-TestAcidGuarantees,,1731751082172.d9e28756f78b0555a6733ffbf6ab98bd.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/d9e28756f78b0555a6733ffbf6ab98bd/A/97d7157baee74526a5cc5386bac01e62 to hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/archive/data/default/TestAcidGuarantees/d9e28756f78b0555a6733ffbf6ab98bd/A/97d7157baee74526a5cc5386bac01e62 2024-11-16T09:58:28,193 DEBUG [StoreCloser-TestAcidGuarantees,,1731751082172.d9e28756f78b0555a6733ffbf6ab98bd.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/d9e28756f78b0555a6733ffbf6ab98bd/B/90cd50f304d648e2bb3f087e808ef5ce, hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/d9e28756f78b0555a6733ffbf6ab98bd/B/c21d5ad86bf84c7f9d79b1232a65de8b, hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/d9e28756f78b0555a6733ffbf6ab98bd/B/c986b11797e945d9b684c5e0c8adb5ef, hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/d9e28756f78b0555a6733ffbf6ab98bd/B/983f5ee278db4f3eaac85d7f97591fda, hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/d9e28756f78b0555a6733ffbf6ab98bd/B/15795d4f2c0e4fdc8b945a6f8860699b, hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/d9e28756f78b0555a6733ffbf6ab98bd/B/284e61e32b8d466384dd3636d4d4fbb6, hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/d9e28756f78b0555a6733ffbf6ab98bd/B/52c98c4ef2174fe3879c8bf25ea45439, hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/d9e28756f78b0555a6733ffbf6ab98bd/B/4d7c8b1b9e844592bd35ef66f2d8167b, hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/d9e28756f78b0555a6733ffbf6ab98bd/B/706a630aaba4416ba4ad59882a64793c, hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/d9e28756f78b0555a6733ffbf6ab98bd/B/e3e6fb7a08bc4bbda4fbb1276b27aba1, hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/d9e28756f78b0555a6733ffbf6ab98bd/B/ea425dc15ba4426e82f075375750f365, hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/d9e28756f78b0555a6733ffbf6ab98bd/B/15113ec6a9d84591914d35a99db57b4d, hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/d9e28756f78b0555a6733ffbf6ab98bd/B/481575664e91483baf55d8b39ada90f2, hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/d9e28756f78b0555a6733ffbf6ab98bd/B/1239686f4cbd48f49490f5231be68ce7, hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/d9e28756f78b0555a6733ffbf6ab98bd/B/2fbcc110db0147edba11d9d86dc65884, hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/d9e28756f78b0555a6733ffbf6ab98bd/B/a93e2c2d05e94cf0b36912ca7912192a, hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/d9e28756f78b0555a6733ffbf6ab98bd/B/382cc9e0996747efbc235a1970fd6c61, hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/d9e28756f78b0555a6733ffbf6ab98bd/B/836a5918e6164bf5a0d6607cf30b4271, hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/d9e28756f78b0555a6733ffbf6ab98bd/B/a73b24e65a43405b9f959afc6671a804, hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/d9e28756f78b0555a6733ffbf6ab98bd/B/8a0b6de738244977a6688f5d91d6d3dc, hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/d9e28756f78b0555a6733ffbf6ab98bd/B/0d5fa0a41a674039ac8d14c4f2a5924a, hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/d9e28756f78b0555a6733ffbf6ab98bd/B/5981e9c45ed3447bacfe73467c7101e9, hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/d9e28756f78b0555a6733ffbf6ab98bd/B/b468b9f5c55b4a87ad1e6ee8500a6260, hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/d9e28756f78b0555a6733ffbf6ab98bd/B/bd5c80c59d6c481191c37eb5ef911443, hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/d9e28756f78b0555a6733ffbf6ab98bd/B/3fade5830bae4cc28018c7702bc3b811, hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/d9e28756f78b0555a6733ffbf6ab98bd/B/948594f9b3c641f28c66b71ad0b18386, hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/d9e28756f78b0555a6733ffbf6ab98bd/B/a1d07b488d594516be54d1ab5a5ccf78, hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/d9e28756f78b0555a6733ffbf6ab98bd/B/3ef734138e094b948fd60c7570996e0d, hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/d9e28756f78b0555a6733ffbf6ab98bd/B/b63e28d4eb564c249b118b3622ecad37, hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/d9e28756f78b0555a6733ffbf6ab98bd/B/35e2c5c8651a4291bbb7092d0b067dac, hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/d9e28756f78b0555a6733ffbf6ab98bd/B/8f4123247f9e45bc824d823ca5119588, hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/d9e28756f78b0555a6733ffbf6ab98bd/B/7c5a81154f154146ac304195f673f3de, hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/d9e28756f78b0555a6733ffbf6ab98bd/B/65e44fb87deb47318c5b3f3350ad90bc, hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/d9e28756f78b0555a6733ffbf6ab98bd/B/bc86639a0a354f0aaec9fc1c395896b8, hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/d9e28756f78b0555a6733ffbf6ab98bd/B/d738eb57280e4a3d87fdad85e5a6abb4, hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/d9e28756f78b0555a6733ffbf6ab98bd/B/d8e6fbedab5348f9b8cd16f7b9ca02b9, hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/d9e28756f78b0555a6733ffbf6ab98bd/B/7ad96d41e26d439699f66cb611331ec1, hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/d9e28756f78b0555a6733ffbf6ab98bd/B/eedc3a6977eb4d788be5bede3825efa6, hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/d9e28756f78b0555a6733ffbf6ab98bd/B/e600d8b124904d24bf48bfa690411916, hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/d9e28756f78b0555a6733ffbf6ab98bd/B/cbaff5393ba343358c22e385d872f050, hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/d9e28756f78b0555a6733ffbf6ab98bd/B/419b545776e44644a2c05849328bb1c5] to archive 2024-11-16T09:58:28,195 DEBUG [StoreCloser-TestAcidGuarantees,,1731751082172.d9e28756f78b0555a6733ffbf6ab98bd.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-16T09:58:28,197 DEBUG [StoreCloser-TestAcidGuarantees,,1731751082172.d9e28756f78b0555a6733ffbf6ab98bd.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/d9e28756f78b0555a6733ffbf6ab98bd/B/90cd50f304d648e2bb3f087e808ef5ce to hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/archive/data/default/TestAcidGuarantees/d9e28756f78b0555a6733ffbf6ab98bd/B/90cd50f304d648e2bb3f087e808ef5ce 2024-11-16T09:58:28,198 DEBUG [StoreCloser-TestAcidGuarantees,,1731751082172.d9e28756f78b0555a6733ffbf6ab98bd.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/d9e28756f78b0555a6733ffbf6ab98bd/B/c21d5ad86bf84c7f9d79b1232a65de8b to hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/archive/data/default/TestAcidGuarantees/d9e28756f78b0555a6733ffbf6ab98bd/B/c21d5ad86bf84c7f9d79b1232a65de8b 2024-11-16T09:58:28,199 DEBUG [StoreCloser-TestAcidGuarantees,,1731751082172.d9e28756f78b0555a6733ffbf6ab98bd.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/d9e28756f78b0555a6733ffbf6ab98bd/B/c986b11797e945d9b684c5e0c8adb5ef to hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/archive/data/default/TestAcidGuarantees/d9e28756f78b0555a6733ffbf6ab98bd/B/c986b11797e945d9b684c5e0c8adb5ef 2024-11-16T09:58:28,201 DEBUG [StoreCloser-TestAcidGuarantees,,1731751082172.d9e28756f78b0555a6733ffbf6ab98bd.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/d9e28756f78b0555a6733ffbf6ab98bd/B/983f5ee278db4f3eaac85d7f97591fda to hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/archive/data/default/TestAcidGuarantees/d9e28756f78b0555a6733ffbf6ab98bd/B/983f5ee278db4f3eaac85d7f97591fda 2024-11-16T09:58:28,202 DEBUG [StoreCloser-TestAcidGuarantees,,1731751082172.d9e28756f78b0555a6733ffbf6ab98bd.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/d9e28756f78b0555a6733ffbf6ab98bd/B/15795d4f2c0e4fdc8b945a6f8860699b to hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/archive/data/default/TestAcidGuarantees/d9e28756f78b0555a6733ffbf6ab98bd/B/15795d4f2c0e4fdc8b945a6f8860699b 2024-11-16T09:58:28,203 DEBUG [StoreCloser-TestAcidGuarantees,,1731751082172.d9e28756f78b0555a6733ffbf6ab98bd.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/d9e28756f78b0555a6733ffbf6ab98bd/B/284e61e32b8d466384dd3636d4d4fbb6 to hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/archive/data/default/TestAcidGuarantees/d9e28756f78b0555a6733ffbf6ab98bd/B/284e61e32b8d466384dd3636d4d4fbb6 2024-11-16T09:58:28,205 DEBUG [StoreCloser-TestAcidGuarantees,,1731751082172.d9e28756f78b0555a6733ffbf6ab98bd.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/d9e28756f78b0555a6733ffbf6ab98bd/B/52c98c4ef2174fe3879c8bf25ea45439 to hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/archive/data/default/TestAcidGuarantees/d9e28756f78b0555a6733ffbf6ab98bd/B/52c98c4ef2174fe3879c8bf25ea45439 2024-11-16T09:58:28,206 DEBUG [StoreCloser-TestAcidGuarantees,,1731751082172.d9e28756f78b0555a6733ffbf6ab98bd.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/d9e28756f78b0555a6733ffbf6ab98bd/B/4d7c8b1b9e844592bd35ef66f2d8167b to hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/archive/data/default/TestAcidGuarantees/d9e28756f78b0555a6733ffbf6ab98bd/B/4d7c8b1b9e844592bd35ef66f2d8167b 2024-11-16T09:58:28,207 DEBUG [StoreCloser-TestAcidGuarantees,,1731751082172.d9e28756f78b0555a6733ffbf6ab98bd.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/d9e28756f78b0555a6733ffbf6ab98bd/B/706a630aaba4416ba4ad59882a64793c to hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/archive/data/default/TestAcidGuarantees/d9e28756f78b0555a6733ffbf6ab98bd/B/706a630aaba4416ba4ad59882a64793c 2024-11-16T09:58:28,208 DEBUG [StoreCloser-TestAcidGuarantees,,1731751082172.d9e28756f78b0555a6733ffbf6ab98bd.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/d9e28756f78b0555a6733ffbf6ab98bd/B/e3e6fb7a08bc4bbda4fbb1276b27aba1 to hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/archive/data/default/TestAcidGuarantees/d9e28756f78b0555a6733ffbf6ab98bd/B/e3e6fb7a08bc4bbda4fbb1276b27aba1 2024-11-16T09:58:28,209 DEBUG [StoreCloser-TestAcidGuarantees,,1731751082172.d9e28756f78b0555a6733ffbf6ab98bd.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/d9e28756f78b0555a6733ffbf6ab98bd/B/ea425dc15ba4426e82f075375750f365 to hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/archive/data/default/TestAcidGuarantees/d9e28756f78b0555a6733ffbf6ab98bd/B/ea425dc15ba4426e82f075375750f365 2024-11-16T09:58:28,211 DEBUG [StoreCloser-TestAcidGuarantees,,1731751082172.d9e28756f78b0555a6733ffbf6ab98bd.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/d9e28756f78b0555a6733ffbf6ab98bd/B/15113ec6a9d84591914d35a99db57b4d to hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/archive/data/default/TestAcidGuarantees/d9e28756f78b0555a6733ffbf6ab98bd/B/15113ec6a9d84591914d35a99db57b4d 2024-11-16T09:58:28,212 DEBUG [StoreCloser-TestAcidGuarantees,,1731751082172.d9e28756f78b0555a6733ffbf6ab98bd.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/d9e28756f78b0555a6733ffbf6ab98bd/B/481575664e91483baf55d8b39ada90f2 to hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/archive/data/default/TestAcidGuarantees/d9e28756f78b0555a6733ffbf6ab98bd/B/481575664e91483baf55d8b39ada90f2 2024-11-16T09:58:28,213 DEBUG [StoreCloser-TestAcidGuarantees,,1731751082172.d9e28756f78b0555a6733ffbf6ab98bd.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/d9e28756f78b0555a6733ffbf6ab98bd/B/1239686f4cbd48f49490f5231be68ce7 to hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/archive/data/default/TestAcidGuarantees/d9e28756f78b0555a6733ffbf6ab98bd/B/1239686f4cbd48f49490f5231be68ce7 2024-11-16T09:58:28,214 DEBUG [StoreCloser-TestAcidGuarantees,,1731751082172.d9e28756f78b0555a6733ffbf6ab98bd.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/d9e28756f78b0555a6733ffbf6ab98bd/B/2fbcc110db0147edba11d9d86dc65884 to hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/archive/data/default/TestAcidGuarantees/d9e28756f78b0555a6733ffbf6ab98bd/B/2fbcc110db0147edba11d9d86dc65884 2024-11-16T09:58:28,215 DEBUG [StoreCloser-TestAcidGuarantees,,1731751082172.d9e28756f78b0555a6733ffbf6ab98bd.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/d9e28756f78b0555a6733ffbf6ab98bd/B/a93e2c2d05e94cf0b36912ca7912192a to hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/archive/data/default/TestAcidGuarantees/d9e28756f78b0555a6733ffbf6ab98bd/B/a93e2c2d05e94cf0b36912ca7912192a 2024-11-16T09:58:28,216 DEBUG [StoreCloser-TestAcidGuarantees,,1731751082172.d9e28756f78b0555a6733ffbf6ab98bd.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/d9e28756f78b0555a6733ffbf6ab98bd/B/382cc9e0996747efbc235a1970fd6c61 to hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/archive/data/default/TestAcidGuarantees/d9e28756f78b0555a6733ffbf6ab98bd/B/382cc9e0996747efbc235a1970fd6c61 2024-11-16T09:58:28,217 DEBUG [StoreCloser-TestAcidGuarantees,,1731751082172.d9e28756f78b0555a6733ffbf6ab98bd.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/d9e28756f78b0555a6733ffbf6ab98bd/B/836a5918e6164bf5a0d6607cf30b4271 to hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/archive/data/default/TestAcidGuarantees/d9e28756f78b0555a6733ffbf6ab98bd/B/836a5918e6164bf5a0d6607cf30b4271 2024-11-16T09:58:28,218 DEBUG [StoreCloser-TestAcidGuarantees,,1731751082172.d9e28756f78b0555a6733ffbf6ab98bd.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/d9e28756f78b0555a6733ffbf6ab98bd/B/a73b24e65a43405b9f959afc6671a804 to hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/archive/data/default/TestAcidGuarantees/d9e28756f78b0555a6733ffbf6ab98bd/B/a73b24e65a43405b9f959afc6671a804 2024-11-16T09:58:28,219 DEBUG [StoreCloser-TestAcidGuarantees,,1731751082172.d9e28756f78b0555a6733ffbf6ab98bd.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/d9e28756f78b0555a6733ffbf6ab98bd/B/8a0b6de738244977a6688f5d91d6d3dc to hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/archive/data/default/TestAcidGuarantees/d9e28756f78b0555a6733ffbf6ab98bd/B/8a0b6de738244977a6688f5d91d6d3dc 2024-11-16T09:58:28,221 DEBUG [StoreCloser-TestAcidGuarantees,,1731751082172.d9e28756f78b0555a6733ffbf6ab98bd.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/d9e28756f78b0555a6733ffbf6ab98bd/B/0d5fa0a41a674039ac8d14c4f2a5924a to hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/archive/data/default/TestAcidGuarantees/d9e28756f78b0555a6733ffbf6ab98bd/B/0d5fa0a41a674039ac8d14c4f2a5924a 2024-11-16T09:58:28,222 DEBUG [StoreCloser-TestAcidGuarantees,,1731751082172.d9e28756f78b0555a6733ffbf6ab98bd.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/d9e28756f78b0555a6733ffbf6ab98bd/B/5981e9c45ed3447bacfe73467c7101e9 to hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/archive/data/default/TestAcidGuarantees/d9e28756f78b0555a6733ffbf6ab98bd/B/5981e9c45ed3447bacfe73467c7101e9 2024-11-16T09:58:28,223 DEBUG [StoreCloser-TestAcidGuarantees,,1731751082172.d9e28756f78b0555a6733ffbf6ab98bd.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/d9e28756f78b0555a6733ffbf6ab98bd/B/b468b9f5c55b4a87ad1e6ee8500a6260 to hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/archive/data/default/TestAcidGuarantees/d9e28756f78b0555a6733ffbf6ab98bd/B/b468b9f5c55b4a87ad1e6ee8500a6260 2024-11-16T09:58:28,225 DEBUG [StoreCloser-TestAcidGuarantees,,1731751082172.d9e28756f78b0555a6733ffbf6ab98bd.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/d9e28756f78b0555a6733ffbf6ab98bd/B/bd5c80c59d6c481191c37eb5ef911443 to hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/archive/data/default/TestAcidGuarantees/d9e28756f78b0555a6733ffbf6ab98bd/B/bd5c80c59d6c481191c37eb5ef911443 2024-11-16T09:58:28,226 DEBUG [StoreCloser-TestAcidGuarantees,,1731751082172.d9e28756f78b0555a6733ffbf6ab98bd.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/d9e28756f78b0555a6733ffbf6ab98bd/B/3fade5830bae4cc28018c7702bc3b811 to hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/archive/data/default/TestAcidGuarantees/d9e28756f78b0555a6733ffbf6ab98bd/B/3fade5830bae4cc28018c7702bc3b811 2024-11-16T09:58:28,227 DEBUG [StoreCloser-TestAcidGuarantees,,1731751082172.d9e28756f78b0555a6733ffbf6ab98bd.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/d9e28756f78b0555a6733ffbf6ab98bd/B/948594f9b3c641f28c66b71ad0b18386 to hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/archive/data/default/TestAcidGuarantees/d9e28756f78b0555a6733ffbf6ab98bd/B/948594f9b3c641f28c66b71ad0b18386 2024-11-16T09:58:28,228 DEBUG [StoreCloser-TestAcidGuarantees,,1731751082172.d9e28756f78b0555a6733ffbf6ab98bd.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/d9e28756f78b0555a6733ffbf6ab98bd/B/a1d07b488d594516be54d1ab5a5ccf78 to hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/archive/data/default/TestAcidGuarantees/d9e28756f78b0555a6733ffbf6ab98bd/B/a1d07b488d594516be54d1ab5a5ccf78 2024-11-16T09:58:28,229 DEBUG [StoreCloser-TestAcidGuarantees,,1731751082172.d9e28756f78b0555a6733ffbf6ab98bd.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/d9e28756f78b0555a6733ffbf6ab98bd/B/3ef734138e094b948fd60c7570996e0d to hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/archive/data/default/TestAcidGuarantees/d9e28756f78b0555a6733ffbf6ab98bd/B/3ef734138e094b948fd60c7570996e0d 2024-11-16T09:58:28,230 DEBUG [StoreCloser-TestAcidGuarantees,,1731751082172.d9e28756f78b0555a6733ffbf6ab98bd.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/d9e28756f78b0555a6733ffbf6ab98bd/B/b63e28d4eb564c249b118b3622ecad37 to hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/archive/data/default/TestAcidGuarantees/d9e28756f78b0555a6733ffbf6ab98bd/B/b63e28d4eb564c249b118b3622ecad37 2024-11-16T09:58:28,231 DEBUG [StoreCloser-TestAcidGuarantees,,1731751082172.d9e28756f78b0555a6733ffbf6ab98bd.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/d9e28756f78b0555a6733ffbf6ab98bd/B/35e2c5c8651a4291bbb7092d0b067dac to hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/archive/data/default/TestAcidGuarantees/d9e28756f78b0555a6733ffbf6ab98bd/B/35e2c5c8651a4291bbb7092d0b067dac 2024-11-16T09:58:28,232 DEBUG [StoreCloser-TestAcidGuarantees,,1731751082172.d9e28756f78b0555a6733ffbf6ab98bd.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/d9e28756f78b0555a6733ffbf6ab98bd/B/8f4123247f9e45bc824d823ca5119588 to hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/archive/data/default/TestAcidGuarantees/d9e28756f78b0555a6733ffbf6ab98bd/B/8f4123247f9e45bc824d823ca5119588 2024-11-16T09:58:28,233 DEBUG [StoreCloser-TestAcidGuarantees,,1731751082172.d9e28756f78b0555a6733ffbf6ab98bd.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/d9e28756f78b0555a6733ffbf6ab98bd/B/7c5a81154f154146ac304195f673f3de to hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/archive/data/default/TestAcidGuarantees/d9e28756f78b0555a6733ffbf6ab98bd/B/7c5a81154f154146ac304195f673f3de 2024-11-16T09:58:28,235 DEBUG [StoreCloser-TestAcidGuarantees,,1731751082172.d9e28756f78b0555a6733ffbf6ab98bd.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/d9e28756f78b0555a6733ffbf6ab98bd/B/65e44fb87deb47318c5b3f3350ad90bc to hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/archive/data/default/TestAcidGuarantees/d9e28756f78b0555a6733ffbf6ab98bd/B/65e44fb87deb47318c5b3f3350ad90bc 2024-11-16T09:58:28,236 DEBUG [StoreCloser-TestAcidGuarantees,,1731751082172.d9e28756f78b0555a6733ffbf6ab98bd.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/d9e28756f78b0555a6733ffbf6ab98bd/B/bc86639a0a354f0aaec9fc1c395896b8 to hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/archive/data/default/TestAcidGuarantees/d9e28756f78b0555a6733ffbf6ab98bd/B/bc86639a0a354f0aaec9fc1c395896b8 2024-11-16T09:58:28,237 DEBUG [StoreCloser-TestAcidGuarantees,,1731751082172.d9e28756f78b0555a6733ffbf6ab98bd.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/d9e28756f78b0555a6733ffbf6ab98bd/B/d738eb57280e4a3d87fdad85e5a6abb4 to hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/archive/data/default/TestAcidGuarantees/d9e28756f78b0555a6733ffbf6ab98bd/B/d738eb57280e4a3d87fdad85e5a6abb4 2024-11-16T09:58:28,238 DEBUG [StoreCloser-TestAcidGuarantees,,1731751082172.d9e28756f78b0555a6733ffbf6ab98bd.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/d9e28756f78b0555a6733ffbf6ab98bd/B/d8e6fbedab5348f9b8cd16f7b9ca02b9 to hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/archive/data/default/TestAcidGuarantees/d9e28756f78b0555a6733ffbf6ab98bd/B/d8e6fbedab5348f9b8cd16f7b9ca02b9 2024-11-16T09:58:28,239 DEBUG [StoreCloser-TestAcidGuarantees,,1731751082172.d9e28756f78b0555a6733ffbf6ab98bd.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/d9e28756f78b0555a6733ffbf6ab98bd/B/7ad96d41e26d439699f66cb611331ec1 to hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/archive/data/default/TestAcidGuarantees/d9e28756f78b0555a6733ffbf6ab98bd/B/7ad96d41e26d439699f66cb611331ec1 2024-11-16T09:58:28,240 DEBUG [StoreCloser-TestAcidGuarantees,,1731751082172.d9e28756f78b0555a6733ffbf6ab98bd.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/d9e28756f78b0555a6733ffbf6ab98bd/B/eedc3a6977eb4d788be5bede3825efa6 to hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/archive/data/default/TestAcidGuarantees/d9e28756f78b0555a6733ffbf6ab98bd/B/eedc3a6977eb4d788be5bede3825efa6 2024-11-16T09:58:28,241 DEBUG [StoreCloser-TestAcidGuarantees,,1731751082172.d9e28756f78b0555a6733ffbf6ab98bd.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/d9e28756f78b0555a6733ffbf6ab98bd/B/e600d8b124904d24bf48bfa690411916 to hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/archive/data/default/TestAcidGuarantees/d9e28756f78b0555a6733ffbf6ab98bd/B/e600d8b124904d24bf48bfa690411916 2024-11-16T09:58:28,242 DEBUG [StoreCloser-TestAcidGuarantees,,1731751082172.d9e28756f78b0555a6733ffbf6ab98bd.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/d9e28756f78b0555a6733ffbf6ab98bd/B/cbaff5393ba343358c22e385d872f050 to hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/archive/data/default/TestAcidGuarantees/d9e28756f78b0555a6733ffbf6ab98bd/B/cbaff5393ba343358c22e385d872f050 2024-11-16T09:58:28,243 DEBUG [StoreCloser-TestAcidGuarantees,,1731751082172.d9e28756f78b0555a6733ffbf6ab98bd.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/d9e28756f78b0555a6733ffbf6ab98bd/B/419b545776e44644a2c05849328bb1c5 to hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/archive/data/default/TestAcidGuarantees/d9e28756f78b0555a6733ffbf6ab98bd/B/419b545776e44644a2c05849328bb1c5 2024-11-16T09:58:28,245 DEBUG [StoreCloser-TestAcidGuarantees,,1731751082172.d9e28756f78b0555a6733ffbf6ab98bd.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/d9e28756f78b0555a6733ffbf6ab98bd/C/0eabc0bd610f4b8fb928684c5386127e, hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/d9e28756f78b0555a6733ffbf6ab98bd/C/1537924938324874ace183a7fc8c33fb, hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/d9e28756f78b0555a6733ffbf6ab98bd/C/4828c9b1f4464dccaaa081ce503fa018, hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/d9e28756f78b0555a6733ffbf6ab98bd/C/53be78a7cd2a4f1d98224b1c2affc854, hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/d9e28756f78b0555a6733ffbf6ab98bd/C/7e5b4afd436c42ddaeae1ac5915abcc6, hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/d9e28756f78b0555a6733ffbf6ab98bd/C/aba8c9e7079440bebfb145d1604cd540, hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/d9e28756f78b0555a6733ffbf6ab98bd/C/8ffd381310ac430bb3fa37b8738b0544, hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/d9e28756f78b0555a6733ffbf6ab98bd/C/775aa7ebdf894f119f74fcbc7d28ea62, hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/d9e28756f78b0555a6733ffbf6ab98bd/C/7a3f8766b7834cda80087a8ec71ec9e7, hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/d9e28756f78b0555a6733ffbf6ab98bd/C/43f98f5a3e8c418080fd6b081c369f71, hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/d9e28756f78b0555a6733ffbf6ab98bd/C/bc833c50bb2540ac9dda563506ca1e2a, hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/d9e28756f78b0555a6733ffbf6ab98bd/C/3ca5b8889378477bb2737c33c064ccc2, hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/d9e28756f78b0555a6733ffbf6ab98bd/C/d6255616e9f941a9b2b01d944bbf4af9, hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/d9e28756f78b0555a6733ffbf6ab98bd/C/2ae6fc50bfb0440580104fede526210d, hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/d9e28756f78b0555a6733ffbf6ab98bd/C/53189d6164424f0aa851310b40b43cbb, hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/d9e28756f78b0555a6733ffbf6ab98bd/C/79c45b5c5f474c3ea9417e42d3647139, hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/d9e28756f78b0555a6733ffbf6ab98bd/C/525e25af8af54a7a8646f1ffeb98e1d5, hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/d9e28756f78b0555a6733ffbf6ab98bd/C/c8dd3aac45dc4fd3a068c59808acb2c5, hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/d9e28756f78b0555a6733ffbf6ab98bd/C/36beb8f139a642b6b398f3dff65758fb, hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/d9e28756f78b0555a6733ffbf6ab98bd/C/8b9ac2a67ae34acfab4cd9dcd7d9e194, hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/d9e28756f78b0555a6733ffbf6ab98bd/C/5058b1fc65bf44d6b262d1e0e572bc03, hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/d9e28756f78b0555a6733ffbf6ab98bd/C/d869628b6caf43adb4766aab62057038, hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/d9e28756f78b0555a6733ffbf6ab98bd/C/167e57611134426aa91dd1a849a17aae, hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/d9e28756f78b0555a6733ffbf6ab98bd/C/be1fe7596ca44a539b964b891050ec6f, hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/d9e28756f78b0555a6733ffbf6ab98bd/C/0e51d1c5bd2e4038af191f3febcb368a, hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/d9e28756f78b0555a6733ffbf6ab98bd/C/98262bb2b6fb41ecbf6e44ee204fa711, hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/d9e28756f78b0555a6733ffbf6ab98bd/C/bafc286da4bb4b178dc9fe040d3351f4, hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/d9e28756f78b0555a6733ffbf6ab98bd/C/82d83ddf384a4de2b8ba2f34f81b7bca, hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/d9e28756f78b0555a6733ffbf6ab98bd/C/6abf151a60c5402f91aa7af92dc02406, hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/d9e28756f78b0555a6733ffbf6ab98bd/C/0f9fc7e9d0654382b9999d87e440edee, hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/d9e28756f78b0555a6733ffbf6ab98bd/C/d71cf2b421514dbf9ac92fd09d0b6add, hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/d9e28756f78b0555a6733ffbf6ab98bd/C/191b9d45ffa9453395b379bdb72884b4, hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/d9e28756f78b0555a6733ffbf6ab98bd/C/ec2fc03b717144878441c4f35d7afb67, hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/d9e28756f78b0555a6733ffbf6ab98bd/C/95c0d63049154ac794ce470e2eb2ed66, hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/d9e28756f78b0555a6733ffbf6ab98bd/C/d9504800b71f4da08755d6f66cbed8ab, hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/d9e28756f78b0555a6733ffbf6ab98bd/C/3cfb8ef2898847a385d61a4e27bcc4b7, hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/d9e28756f78b0555a6733ffbf6ab98bd/C/6513ccc313dc49b28664372e3f14691a, hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/d9e28756f78b0555a6733ffbf6ab98bd/C/f5224569676a4236b43ae5c1ebc30ddd, hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/d9e28756f78b0555a6733ffbf6ab98bd/C/a740d4a75fb5467784531979bb9dda59, hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/d9e28756f78b0555a6733ffbf6ab98bd/C/510302ea2afd46e599b5647484a585b6, hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/d9e28756f78b0555a6733ffbf6ab98bd/C/fbeb8c2fb73f49599879a72b756eaeb1] to archive 2024-11-16T09:58:28,246 DEBUG [StoreCloser-TestAcidGuarantees,,1731751082172.d9e28756f78b0555a6733ffbf6ab98bd.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-16T09:58:28,248 DEBUG [StoreCloser-TestAcidGuarantees,,1731751082172.d9e28756f78b0555a6733ffbf6ab98bd.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/d9e28756f78b0555a6733ffbf6ab98bd/C/0eabc0bd610f4b8fb928684c5386127e to hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/archive/data/default/TestAcidGuarantees/d9e28756f78b0555a6733ffbf6ab98bd/C/0eabc0bd610f4b8fb928684c5386127e 2024-11-16T09:58:28,249 DEBUG [StoreCloser-TestAcidGuarantees,,1731751082172.d9e28756f78b0555a6733ffbf6ab98bd.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/d9e28756f78b0555a6733ffbf6ab98bd/C/1537924938324874ace183a7fc8c33fb to hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/archive/data/default/TestAcidGuarantees/d9e28756f78b0555a6733ffbf6ab98bd/C/1537924938324874ace183a7fc8c33fb 2024-11-16T09:58:28,250 DEBUG [StoreCloser-TestAcidGuarantees,,1731751082172.d9e28756f78b0555a6733ffbf6ab98bd.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/d9e28756f78b0555a6733ffbf6ab98bd/C/4828c9b1f4464dccaaa081ce503fa018 to hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/archive/data/default/TestAcidGuarantees/d9e28756f78b0555a6733ffbf6ab98bd/C/4828c9b1f4464dccaaa081ce503fa018 2024-11-16T09:58:28,251 DEBUG [StoreCloser-TestAcidGuarantees,,1731751082172.d9e28756f78b0555a6733ffbf6ab98bd.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/d9e28756f78b0555a6733ffbf6ab98bd/C/53be78a7cd2a4f1d98224b1c2affc854 to hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/archive/data/default/TestAcidGuarantees/d9e28756f78b0555a6733ffbf6ab98bd/C/53be78a7cd2a4f1d98224b1c2affc854 2024-11-16T09:58:28,252 DEBUG [StoreCloser-TestAcidGuarantees,,1731751082172.d9e28756f78b0555a6733ffbf6ab98bd.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/d9e28756f78b0555a6733ffbf6ab98bd/C/7e5b4afd436c42ddaeae1ac5915abcc6 to hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/archive/data/default/TestAcidGuarantees/d9e28756f78b0555a6733ffbf6ab98bd/C/7e5b4afd436c42ddaeae1ac5915abcc6 2024-11-16T09:58:28,253 DEBUG [StoreCloser-TestAcidGuarantees,,1731751082172.d9e28756f78b0555a6733ffbf6ab98bd.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/d9e28756f78b0555a6733ffbf6ab98bd/C/aba8c9e7079440bebfb145d1604cd540 to hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/archive/data/default/TestAcidGuarantees/d9e28756f78b0555a6733ffbf6ab98bd/C/aba8c9e7079440bebfb145d1604cd540 2024-11-16T09:58:28,254 DEBUG [StoreCloser-TestAcidGuarantees,,1731751082172.d9e28756f78b0555a6733ffbf6ab98bd.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/d9e28756f78b0555a6733ffbf6ab98bd/C/8ffd381310ac430bb3fa37b8738b0544 to hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/archive/data/default/TestAcidGuarantees/d9e28756f78b0555a6733ffbf6ab98bd/C/8ffd381310ac430bb3fa37b8738b0544 2024-11-16T09:58:28,255 DEBUG [StoreCloser-TestAcidGuarantees,,1731751082172.d9e28756f78b0555a6733ffbf6ab98bd.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/d9e28756f78b0555a6733ffbf6ab98bd/C/775aa7ebdf894f119f74fcbc7d28ea62 to hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/archive/data/default/TestAcidGuarantees/d9e28756f78b0555a6733ffbf6ab98bd/C/775aa7ebdf894f119f74fcbc7d28ea62 2024-11-16T09:58:28,257 DEBUG [StoreCloser-TestAcidGuarantees,,1731751082172.d9e28756f78b0555a6733ffbf6ab98bd.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/d9e28756f78b0555a6733ffbf6ab98bd/C/7a3f8766b7834cda80087a8ec71ec9e7 to hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/archive/data/default/TestAcidGuarantees/d9e28756f78b0555a6733ffbf6ab98bd/C/7a3f8766b7834cda80087a8ec71ec9e7 2024-11-16T09:58:28,258 DEBUG [StoreCloser-TestAcidGuarantees,,1731751082172.d9e28756f78b0555a6733ffbf6ab98bd.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/d9e28756f78b0555a6733ffbf6ab98bd/C/43f98f5a3e8c418080fd6b081c369f71 to hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/archive/data/default/TestAcidGuarantees/d9e28756f78b0555a6733ffbf6ab98bd/C/43f98f5a3e8c418080fd6b081c369f71 2024-11-16T09:58:28,259 DEBUG [StoreCloser-TestAcidGuarantees,,1731751082172.d9e28756f78b0555a6733ffbf6ab98bd.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/d9e28756f78b0555a6733ffbf6ab98bd/C/bc833c50bb2540ac9dda563506ca1e2a to hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/archive/data/default/TestAcidGuarantees/d9e28756f78b0555a6733ffbf6ab98bd/C/bc833c50bb2540ac9dda563506ca1e2a 2024-11-16T09:58:28,260 DEBUG [StoreCloser-TestAcidGuarantees,,1731751082172.d9e28756f78b0555a6733ffbf6ab98bd.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/d9e28756f78b0555a6733ffbf6ab98bd/C/3ca5b8889378477bb2737c33c064ccc2 to hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/archive/data/default/TestAcidGuarantees/d9e28756f78b0555a6733ffbf6ab98bd/C/3ca5b8889378477bb2737c33c064ccc2 2024-11-16T09:58:28,261 DEBUG [StoreCloser-TestAcidGuarantees,,1731751082172.d9e28756f78b0555a6733ffbf6ab98bd.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/d9e28756f78b0555a6733ffbf6ab98bd/C/d6255616e9f941a9b2b01d944bbf4af9 to hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/archive/data/default/TestAcidGuarantees/d9e28756f78b0555a6733ffbf6ab98bd/C/d6255616e9f941a9b2b01d944bbf4af9 2024-11-16T09:58:28,262 DEBUG [StoreCloser-TestAcidGuarantees,,1731751082172.d9e28756f78b0555a6733ffbf6ab98bd.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/d9e28756f78b0555a6733ffbf6ab98bd/C/2ae6fc50bfb0440580104fede526210d to hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/archive/data/default/TestAcidGuarantees/d9e28756f78b0555a6733ffbf6ab98bd/C/2ae6fc50bfb0440580104fede526210d 2024-11-16T09:58:28,264 DEBUG [StoreCloser-TestAcidGuarantees,,1731751082172.d9e28756f78b0555a6733ffbf6ab98bd.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/d9e28756f78b0555a6733ffbf6ab98bd/C/53189d6164424f0aa851310b40b43cbb to hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/archive/data/default/TestAcidGuarantees/d9e28756f78b0555a6733ffbf6ab98bd/C/53189d6164424f0aa851310b40b43cbb 2024-11-16T09:58:28,265 DEBUG [StoreCloser-TestAcidGuarantees,,1731751082172.d9e28756f78b0555a6733ffbf6ab98bd.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/d9e28756f78b0555a6733ffbf6ab98bd/C/79c45b5c5f474c3ea9417e42d3647139 to hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/archive/data/default/TestAcidGuarantees/d9e28756f78b0555a6733ffbf6ab98bd/C/79c45b5c5f474c3ea9417e42d3647139 2024-11-16T09:58:28,266 DEBUG [StoreCloser-TestAcidGuarantees,,1731751082172.d9e28756f78b0555a6733ffbf6ab98bd.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/d9e28756f78b0555a6733ffbf6ab98bd/C/525e25af8af54a7a8646f1ffeb98e1d5 to hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/archive/data/default/TestAcidGuarantees/d9e28756f78b0555a6733ffbf6ab98bd/C/525e25af8af54a7a8646f1ffeb98e1d5 2024-11-16T09:58:28,267 DEBUG [StoreCloser-TestAcidGuarantees,,1731751082172.d9e28756f78b0555a6733ffbf6ab98bd.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/d9e28756f78b0555a6733ffbf6ab98bd/C/c8dd3aac45dc4fd3a068c59808acb2c5 to hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/archive/data/default/TestAcidGuarantees/d9e28756f78b0555a6733ffbf6ab98bd/C/c8dd3aac45dc4fd3a068c59808acb2c5 2024-11-16T09:58:28,268 DEBUG [StoreCloser-TestAcidGuarantees,,1731751082172.d9e28756f78b0555a6733ffbf6ab98bd.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/d9e28756f78b0555a6733ffbf6ab98bd/C/36beb8f139a642b6b398f3dff65758fb to hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/archive/data/default/TestAcidGuarantees/d9e28756f78b0555a6733ffbf6ab98bd/C/36beb8f139a642b6b398f3dff65758fb 2024-11-16T09:58:28,270 DEBUG [StoreCloser-TestAcidGuarantees,,1731751082172.d9e28756f78b0555a6733ffbf6ab98bd.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/d9e28756f78b0555a6733ffbf6ab98bd/C/8b9ac2a67ae34acfab4cd9dcd7d9e194 to hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/archive/data/default/TestAcidGuarantees/d9e28756f78b0555a6733ffbf6ab98bd/C/8b9ac2a67ae34acfab4cd9dcd7d9e194 2024-11-16T09:58:28,271 DEBUG [StoreCloser-TestAcidGuarantees,,1731751082172.d9e28756f78b0555a6733ffbf6ab98bd.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/d9e28756f78b0555a6733ffbf6ab98bd/C/5058b1fc65bf44d6b262d1e0e572bc03 to hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/archive/data/default/TestAcidGuarantees/d9e28756f78b0555a6733ffbf6ab98bd/C/5058b1fc65bf44d6b262d1e0e572bc03 2024-11-16T09:58:28,272 DEBUG [StoreCloser-TestAcidGuarantees,,1731751082172.d9e28756f78b0555a6733ffbf6ab98bd.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/d9e28756f78b0555a6733ffbf6ab98bd/C/d869628b6caf43adb4766aab62057038 to hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/archive/data/default/TestAcidGuarantees/d9e28756f78b0555a6733ffbf6ab98bd/C/d869628b6caf43adb4766aab62057038 2024-11-16T09:58:28,273 DEBUG [StoreCloser-TestAcidGuarantees,,1731751082172.d9e28756f78b0555a6733ffbf6ab98bd.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/d9e28756f78b0555a6733ffbf6ab98bd/C/167e57611134426aa91dd1a849a17aae to hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/archive/data/default/TestAcidGuarantees/d9e28756f78b0555a6733ffbf6ab98bd/C/167e57611134426aa91dd1a849a17aae 2024-11-16T09:58:28,274 DEBUG [StoreCloser-TestAcidGuarantees,,1731751082172.d9e28756f78b0555a6733ffbf6ab98bd.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/d9e28756f78b0555a6733ffbf6ab98bd/C/be1fe7596ca44a539b964b891050ec6f to hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/archive/data/default/TestAcidGuarantees/d9e28756f78b0555a6733ffbf6ab98bd/C/be1fe7596ca44a539b964b891050ec6f 2024-11-16T09:58:28,275 DEBUG [StoreCloser-TestAcidGuarantees,,1731751082172.d9e28756f78b0555a6733ffbf6ab98bd.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/d9e28756f78b0555a6733ffbf6ab98bd/C/0e51d1c5bd2e4038af191f3febcb368a to hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/archive/data/default/TestAcidGuarantees/d9e28756f78b0555a6733ffbf6ab98bd/C/0e51d1c5bd2e4038af191f3febcb368a 2024-11-16T09:58:28,277 DEBUG [StoreCloser-TestAcidGuarantees,,1731751082172.d9e28756f78b0555a6733ffbf6ab98bd.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/d9e28756f78b0555a6733ffbf6ab98bd/C/98262bb2b6fb41ecbf6e44ee204fa711 to hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/archive/data/default/TestAcidGuarantees/d9e28756f78b0555a6733ffbf6ab98bd/C/98262bb2b6fb41ecbf6e44ee204fa711 2024-11-16T09:58:28,278 DEBUG [StoreCloser-TestAcidGuarantees,,1731751082172.d9e28756f78b0555a6733ffbf6ab98bd.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/d9e28756f78b0555a6733ffbf6ab98bd/C/bafc286da4bb4b178dc9fe040d3351f4 to hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/archive/data/default/TestAcidGuarantees/d9e28756f78b0555a6733ffbf6ab98bd/C/bafc286da4bb4b178dc9fe040d3351f4 2024-11-16T09:58:28,279 DEBUG [StoreCloser-TestAcidGuarantees,,1731751082172.d9e28756f78b0555a6733ffbf6ab98bd.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/d9e28756f78b0555a6733ffbf6ab98bd/C/82d83ddf384a4de2b8ba2f34f81b7bca to hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/archive/data/default/TestAcidGuarantees/d9e28756f78b0555a6733ffbf6ab98bd/C/82d83ddf384a4de2b8ba2f34f81b7bca 2024-11-16T09:58:28,280 DEBUG [StoreCloser-TestAcidGuarantees,,1731751082172.d9e28756f78b0555a6733ffbf6ab98bd.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/d9e28756f78b0555a6733ffbf6ab98bd/C/6abf151a60c5402f91aa7af92dc02406 to hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/archive/data/default/TestAcidGuarantees/d9e28756f78b0555a6733ffbf6ab98bd/C/6abf151a60c5402f91aa7af92dc02406 2024-11-16T09:58:28,281 DEBUG [StoreCloser-TestAcidGuarantees,,1731751082172.d9e28756f78b0555a6733ffbf6ab98bd.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/d9e28756f78b0555a6733ffbf6ab98bd/C/0f9fc7e9d0654382b9999d87e440edee to hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/archive/data/default/TestAcidGuarantees/d9e28756f78b0555a6733ffbf6ab98bd/C/0f9fc7e9d0654382b9999d87e440edee 2024-11-16T09:58:28,283 DEBUG [StoreCloser-TestAcidGuarantees,,1731751082172.d9e28756f78b0555a6733ffbf6ab98bd.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/d9e28756f78b0555a6733ffbf6ab98bd/C/d71cf2b421514dbf9ac92fd09d0b6add to hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/archive/data/default/TestAcidGuarantees/d9e28756f78b0555a6733ffbf6ab98bd/C/d71cf2b421514dbf9ac92fd09d0b6add 2024-11-16T09:58:28,284 DEBUG [StoreCloser-TestAcidGuarantees,,1731751082172.d9e28756f78b0555a6733ffbf6ab98bd.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/d9e28756f78b0555a6733ffbf6ab98bd/C/191b9d45ffa9453395b379bdb72884b4 to hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/archive/data/default/TestAcidGuarantees/d9e28756f78b0555a6733ffbf6ab98bd/C/191b9d45ffa9453395b379bdb72884b4 2024-11-16T09:58:28,285 DEBUG [StoreCloser-TestAcidGuarantees,,1731751082172.d9e28756f78b0555a6733ffbf6ab98bd.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/d9e28756f78b0555a6733ffbf6ab98bd/C/ec2fc03b717144878441c4f35d7afb67 to hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/archive/data/default/TestAcidGuarantees/d9e28756f78b0555a6733ffbf6ab98bd/C/ec2fc03b717144878441c4f35d7afb67 2024-11-16T09:58:28,286 DEBUG [StoreCloser-TestAcidGuarantees,,1731751082172.d9e28756f78b0555a6733ffbf6ab98bd.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/d9e28756f78b0555a6733ffbf6ab98bd/C/95c0d63049154ac794ce470e2eb2ed66 to hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/archive/data/default/TestAcidGuarantees/d9e28756f78b0555a6733ffbf6ab98bd/C/95c0d63049154ac794ce470e2eb2ed66 2024-11-16T09:58:28,287 DEBUG [StoreCloser-TestAcidGuarantees,,1731751082172.d9e28756f78b0555a6733ffbf6ab98bd.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/d9e28756f78b0555a6733ffbf6ab98bd/C/d9504800b71f4da08755d6f66cbed8ab to hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/archive/data/default/TestAcidGuarantees/d9e28756f78b0555a6733ffbf6ab98bd/C/d9504800b71f4da08755d6f66cbed8ab 2024-11-16T09:58:28,288 DEBUG [StoreCloser-TestAcidGuarantees,,1731751082172.d9e28756f78b0555a6733ffbf6ab98bd.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/d9e28756f78b0555a6733ffbf6ab98bd/C/3cfb8ef2898847a385d61a4e27bcc4b7 to hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/archive/data/default/TestAcidGuarantees/d9e28756f78b0555a6733ffbf6ab98bd/C/3cfb8ef2898847a385d61a4e27bcc4b7 2024-11-16T09:58:28,290 DEBUG [StoreCloser-TestAcidGuarantees,,1731751082172.d9e28756f78b0555a6733ffbf6ab98bd.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/d9e28756f78b0555a6733ffbf6ab98bd/C/6513ccc313dc49b28664372e3f14691a to hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/archive/data/default/TestAcidGuarantees/d9e28756f78b0555a6733ffbf6ab98bd/C/6513ccc313dc49b28664372e3f14691a 2024-11-16T09:58:28,291 DEBUG [StoreCloser-TestAcidGuarantees,,1731751082172.d9e28756f78b0555a6733ffbf6ab98bd.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/d9e28756f78b0555a6733ffbf6ab98bd/C/f5224569676a4236b43ae5c1ebc30ddd to hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/archive/data/default/TestAcidGuarantees/d9e28756f78b0555a6733ffbf6ab98bd/C/f5224569676a4236b43ae5c1ebc30ddd 2024-11-16T09:58:28,292 DEBUG [StoreCloser-TestAcidGuarantees,,1731751082172.d9e28756f78b0555a6733ffbf6ab98bd.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/d9e28756f78b0555a6733ffbf6ab98bd/C/a740d4a75fb5467784531979bb9dda59 to hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/archive/data/default/TestAcidGuarantees/d9e28756f78b0555a6733ffbf6ab98bd/C/a740d4a75fb5467784531979bb9dda59 2024-11-16T09:58:28,293 DEBUG [StoreCloser-TestAcidGuarantees,,1731751082172.d9e28756f78b0555a6733ffbf6ab98bd.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/d9e28756f78b0555a6733ffbf6ab98bd/C/510302ea2afd46e599b5647484a585b6 to hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/archive/data/default/TestAcidGuarantees/d9e28756f78b0555a6733ffbf6ab98bd/C/510302ea2afd46e599b5647484a585b6 2024-11-16T09:58:28,294 DEBUG [StoreCloser-TestAcidGuarantees,,1731751082172.d9e28756f78b0555a6733ffbf6ab98bd.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/d9e28756f78b0555a6733ffbf6ab98bd/C/fbeb8c2fb73f49599879a72b756eaeb1 to hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/archive/data/default/TestAcidGuarantees/d9e28756f78b0555a6733ffbf6ab98bd/C/fbeb8c2fb73f49599879a72b756eaeb1 2024-11-16T09:58:28,300 DEBUG [RS_CLOSE_REGION-regionserver/037c58a19ddb:0-0 {event_type=M_RS_CLOSE_REGION, pid=28}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/d9e28756f78b0555a6733ffbf6ab98bd/recovered.edits/599.seqid, newMaxSeqId=599, maxSeqId=1 2024-11-16T09:58:28,301 INFO [RS_CLOSE_REGION-regionserver/037c58a19ddb:0-0 {event_type=M_RS_CLOSE_REGION, pid=28}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1731751082172.d9e28756f78b0555a6733ffbf6ab98bd. 2024-11-16T09:58:28,301 DEBUG [RS_CLOSE_REGION-regionserver/037c58a19ddb:0-0 {event_type=M_RS_CLOSE_REGION, pid=28}] regionserver.HRegion(1676): Region close journal for d9e28756f78b0555a6733ffbf6ab98bd: Waiting for close lock at 1731751106864Running coprocessor pre-close hooks at 1731751106864Disabling compacts and flushes for region at 1731751106864Disabling writes for close at 1731751106864Obtaining lock to block concurrent updates at 1731751106865 (+1 ms)Preparing flush snapshotting stores in d9e28756f78b0555a6733ffbf6ab98bd at 1731751106865Finished memstore snapshotting TestAcidGuarantees,,1731751082172.d9e28756f78b0555a6733ffbf6ab98bd., syncing WAL and waiting on mvcc, flushsize=dataSize=27480, getHeapSize=72720, getOffHeapSize=0, getCellsCount=600 at 1731751106865Flushing stores of TestAcidGuarantees,,1731751082172.d9e28756f78b0555a6733ffbf6ab98bd. at 1731751106866 (+1 ms)Flushing d9e28756f78b0555a6733ffbf6ab98bd/A: creating writer at 1731751106866Flushing d9e28756f78b0555a6733ffbf6ab98bd/A: appending metadata at 1731751106868 (+2 ms)Flushing d9e28756f78b0555a6733ffbf6ab98bd/A: closing flushed file at 1731751106868Flushing d9e28756f78b0555a6733ffbf6ab98bd/B: creating writer at 1731751107278 (+410 ms)Flushing d9e28756f78b0555a6733ffbf6ab98bd/B: appending metadata at 1731751107282 (+4 ms)Flushing d9e28756f78b0555a6733ffbf6ab98bd/B: closing flushed file at 1731751107282Flushing d9e28756f78b0555a6733ffbf6ab98bd/C: creating writer at 1731751107692 (+410 ms)Flushing d9e28756f78b0555a6733ffbf6ab98bd/C: appending metadata at 1731751107694 (+2 ms)Flushing d9e28756f78b0555a6733ffbf6ab98bd/C: closing flushed file at 1731751107695 (+1 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@434554bc: reopening flushed file at 1731751108104 (+409 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@3324ba43: reopening flushed file at 1731751108110 (+6 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@b8f608a: reopening flushed file at 1731751108115 (+5 ms)Finished flush of dataSize ~26.84 KB/27480, heapSize ~71.02 KB/72720, currentSize=0 B/0 for d9e28756f78b0555a6733ffbf6ab98bd in 1256ms, sequenceid=596, compaction requested=false at 1731751108121 (+6 ms)Writing region close event to WAL at 1731751108296 (+175 ms)Running coprocessor post-close hooks at 1731751108301 (+5 ms)Closed at 1731751108301 2024-11-16T09:58:28,304 INFO [RS_CLOSE_REGION-regionserver/037c58a19ddb:0-0 {event_type=M_RS_CLOSE_REGION, pid=28}] handler.UnassignRegionHandler(157): Closed d9e28756f78b0555a6733ffbf6ab98bd 2024-11-16T09:58:28,304 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=27 updating hbase:meta row=d9e28756f78b0555a6733ffbf6ab98bd, regionState=CLOSED 2024-11-16T09:58:28,306 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=28, ppid=27, state=RUNNABLE, hasLock=false; CloseRegionProcedure d9e28756f78b0555a6733ffbf6ab98bd, server=037c58a19ddb,37267,1731751077844 because future has completed 2024-11-16T09:58:28,308 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=28, resume processing ppid=27 2024-11-16T09:58:28,308 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=28, ppid=27, state=SUCCESS, hasLock=false; CloseRegionProcedure d9e28756f78b0555a6733ffbf6ab98bd, server=037c58a19ddb,37267,1731751077844 in 1.6000 sec 2024-11-16T09:58:28,310 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=27, resume processing ppid=26 2024-11-16T09:58:28,310 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=27, ppid=26, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=d9e28756f78b0555a6733ffbf6ab98bd, UNASSIGN in 1.6050 sec 2024-11-16T09:58:28,312 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=26, resume processing ppid=25 2024-11-16T09:58:28,312 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=26, ppid=25, state=SUCCESS, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees in 1.6120 sec 2024-11-16T09:58:28,313 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731751108313"}]},"ts":"1731751108313"} 2024-11-16T09:58:28,315 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLED in hbase:meta 2024-11-16T09:58:28,315 INFO [PEWorker-5 {}] procedure.DisableTableProcedure(296): Set TestAcidGuarantees to state=DISABLED 2024-11-16T09:58:28,316 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=25, state=SUCCESS, hasLock=false; DisableTableProcedure table=TestAcidGuarantees in 1.6250 sec 2024-11-16T09:58:28,843 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37877 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=25 2024-11-16T09:58:28,843 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DISABLE, Table Name: default:TestAcidGuarantees completed 2024-11-16T09:58:28,846 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37877 {}] master.HMaster$5(2570): Client=jenkins//172.17.0.2 delete TestAcidGuarantees 2024-11-16T09:58:28,850 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37877 {}] procedure2.ProcedureExecutor(1139): Stored pid=29, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees 2024-11-16T09:58:28,851 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(103): Waiting for RIT for pid=29, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-16T09:58:28,852 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37877 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=29 2024-11-16T09:58:28,853 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(115): Deleting regions from filesystem for pid=29, state=RUNNABLE:DELETE_TABLE_CLEAR_FS_LAYOUT, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-16T09:58:28,857 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(131): ARCHIVING hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/d9e28756f78b0555a6733ffbf6ab98bd 2024-11-16T09:58:28,861 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(159): Archiving [FileablePath, hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/d9e28756f78b0555a6733ffbf6ab98bd/A, FileablePath, hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/d9e28756f78b0555a6733ffbf6ab98bd/B, FileablePath, hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/d9e28756f78b0555a6733ffbf6ab98bd/C, FileablePath, hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/d9e28756f78b0555a6733ffbf6ab98bd/recovered.edits] 2024-11-16T09:58:28,865 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/d9e28756f78b0555a6733ffbf6ab98bd/A/5b6f13bb7dd94ef3bb875b1d9068c630 to hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/archive/data/default/TestAcidGuarantees/d9e28756f78b0555a6733ffbf6ab98bd/A/5b6f13bb7dd94ef3bb875b1d9068c630 2024-11-16T09:58:28,866 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/d9e28756f78b0555a6733ffbf6ab98bd/A/a9b2865d03dd45cd95d4c39c597e5f93 to hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/archive/data/default/TestAcidGuarantees/d9e28756f78b0555a6733ffbf6ab98bd/A/a9b2865d03dd45cd95d4c39c597e5f93 2024-11-16T09:58:28,869 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/d9e28756f78b0555a6733ffbf6ab98bd/B/f10c3c606923418e9ae4d3c253f985b5 to hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/archive/data/default/TestAcidGuarantees/d9e28756f78b0555a6733ffbf6ab98bd/B/f10c3c606923418e9ae4d3c253f985b5 2024-11-16T09:58:28,871 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/d9e28756f78b0555a6733ffbf6ab98bd/B/f3ae6851416d4f0195116b01d3822866 to hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/archive/data/default/TestAcidGuarantees/d9e28756f78b0555a6733ffbf6ab98bd/B/f3ae6851416d4f0195116b01d3822866 2024-11-16T09:58:28,873 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/d9e28756f78b0555a6733ffbf6ab98bd/C/9f5f3912f5d94654847f8f9a2d3bc588 to hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/archive/data/default/TestAcidGuarantees/d9e28756f78b0555a6733ffbf6ab98bd/C/9f5f3912f5d94654847f8f9a2d3bc588 2024-11-16T09:58:28,875 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/d9e28756f78b0555a6733ffbf6ab98bd/C/df717d2a0a0149e5a5af7207c2b0a989 to hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/archive/data/default/TestAcidGuarantees/d9e28756f78b0555a6733ffbf6ab98bd/C/df717d2a0a0149e5a5af7207c2b0a989 2024-11-16T09:58:28,878 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/d9e28756f78b0555a6733ffbf6ab98bd/recovered.edits/599.seqid to hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/archive/data/default/TestAcidGuarantees/d9e28756f78b0555a6733ffbf6ab98bd/recovered.edits/599.seqid 2024-11-16T09:58:28,878 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(610): Deleted hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/d9e28756f78b0555a6733ffbf6ab98bd 2024-11-16T09:58:28,878 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(313): Archived TestAcidGuarantees regions 2024-11-16T09:58:28,883 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(120): Deleting regions from META for pid=29, state=RUNNABLE:DELETE_TABLE_REMOVE_FROM_META, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-16T09:58:28,887 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=37267 {}] util.ReflectedFunctionCache(97): Populated cache for org.apache.hadoop.hbase.filter.KeyOnlyFilter in 0ms 2024-11-16T09:58:28,892 WARN [PEWorker-1 {}] procedure.DeleteTableProcedure(370): Deleting some vestigial 1 rows of TestAcidGuarantees from hbase:meta 2024-11-16T09:58:28,922 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(407): Removing 'TestAcidGuarantees' descriptor. 2024-11-16T09:58:28,923 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(126): Deleting assignment state for pid=29, state=RUNNABLE:DELETE_TABLE_UNASSIGN_REGIONS, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-16T09:58:28,923 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(397): Removing 'TestAcidGuarantees' from region states. 2024-11-16T09:58:28,924 DEBUG [PEWorker-1 {}] assignment.RegionStateStore(723): Delete {"totalColumns":1,"row":"TestAcidGuarantees,,1731751082172.d9e28756f78b0555a6733ffbf6ab98bd.","families":{"info":[{"qualifier":"","vlen":0,"tag":[],"timestamp":"1731751108923"}]},"ts":"9223372036854775807"} 2024-11-16T09:58:28,928 INFO [PEWorker-1 {}] assignment.RegionStateStore(562): Deleted 1 regions from META 2024-11-16T09:58:28,928 DEBUG [PEWorker-1 {}] assignment.RegionStateStore(563): Deleted regions: [{ENCODED => d9e28756f78b0555a6733ffbf6ab98bd, NAME => 'TestAcidGuarantees,,1731751082172.d9e28756f78b0555a6733ffbf6ab98bd.', STARTKEY => '', ENDKEY => ''}] 2024-11-16T09:58:28,928 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(401): Marking 'TestAcidGuarantees' as deleted. 2024-11-16T09:58:28,928 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Delete {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":0,"tag":[],"timestamp":"1731751108928"}]},"ts":"9223372036854775807"} 2024-11-16T09:58:28,931 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(867): Deleted table TestAcidGuarantees state from META 2024-11-16T09:58:28,932 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(133): Finished pid=29, state=RUNNABLE:DELETE_TABLE_POST_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-16T09:58:28,933 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=29, state=SUCCESS, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees in 85 msec 2024-11-16T09:58:28,963 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37877 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=29 2024-11-16T09:58:28,963 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocator(219): Clear meta cache for TestAcidGuarantees 2024-11-16T09:58:28,964 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DELETE, Table Name: default:TestAcidGuarantees completed 2024-11-16T09:58:28,977 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: TestAcidGuaranteesWithNoInMemCompaction#testMixedAtomicity Thread=241 (was 211) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-5 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-11 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_OPEN_REGION-regionserver/037c58a19ddb:0-0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_FLUSH_OPERATIONS-regionserver/037c58a19ddb:0-1 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-7 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-10 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_CLOSE_REGION-regionserver/037c58a19ddb:0-0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: HFileArchiver-1 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ProcedureExecutor-Async-Task-Executor-3 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: Timer for 'HBase' metrics system java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.util.TimerThread.mainLoop(Timer.java:563) java.base@17.0.11/java.util.TimerThread.run(Timer.java:516) Potentially hanging thread: ProcedureExecutor-Async-Task-Executor-1 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-14 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-15 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-6 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-791136307_22 at /127.0.0.1:44244 [Waiting for operation #7] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS:0;037c58a19ddb:37267-shortCompactions-0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.PriorityBlockingQueue.take(PriorityBlockingQueue.java:535) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-13 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_FLUSH_OPERATIONS-regionserver/037c58a19ddb:0-2 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RSProcedureDispatcher-pool-2 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-9 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ProcedureExecutor-Async-Task-Executor-0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-8 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-1188164596_22 at /127.0.0.1:41774 [Waiting for operation #381] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-16 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RSProcedureDispatcher-pool-1 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-12 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-4 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:62) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:883) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ProcedureExecutor-Async-Task-Executor-2 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-1188164596_22 at /127.0.0.1:39830 [Waiting for operation #406] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-1188164596_22 at /127.0.0.1:44248 [Waiting for operation #8] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-791136307_22 at /127.0.0.1:44260 [Waiting for operation #7] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_FLUSH_OPERATIONS-regionserver/037c58a19ddb:0-0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=463 (was 441) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=309 (was 223) - SystemLoadAverage LEAK? -, ProcessCount=11 (was 11), AvailableMemoryMB=5349 (was 5911) 2024-11-16T09:58:28,987 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: TestAcidGuaranteesWithNoInMemCompaction#testMobMixedAtomicity Thread=241, OpenFileDescriptor=463, MaxFileDescriptor=1048576, SystemLoadAverage=309, ProcessCount=11, AvailableMemoryMB=5349 2024-11-16T09:58:28,989 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37877 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (131072) is too small, which might cause very frequent flushing. 2024-11-16T09:58:28,989 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37877 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.2 create 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'NONE'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-16T09:58:28,990 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37877 {}] procedure2.ProcedureExecutor(1139): Stored pid=30, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=TestAcidGuarantees 2024-11-16T09:58:28,991 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=30, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_PRE_OPERATION 2024-11-16T09:58:28,992 DEBUG [PEWorker-3 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T09:58:28,992 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37877 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.2 procedure request for creating table: namespace: "default" qualifier: "TestAcidGuarantees" procId is: 30 2024-11-16T09:58:28,993 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37877 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=30 2024-11-16T09:58:28,993 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=30, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-11-16T09:58:28,998 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37591 is added to blk_1073741966_1142 (size=959) 2024-11-16T09:58:29,103 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37877 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=30 2024-11-16T09:58:29,313 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37877 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=30 2024-11-16T09:58:29,401 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => 7c7f87a6124bfe2f4049ee7803e0f98c, NAME => 'TestAcidGuarantees,,1731751108989.7c7f87a6124bfe2f4049ee7803e0f98c.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'NONE', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592 2024-11-16T09:58:29,407 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37591 is added to blk_1073741967_1143 (size=53) 2024-11-16T09:58:29,623 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37877 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=30 2024-11-16T09:58:29,808 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1731751108989.7c7f87a6124bfe2f4049ee7803e0f98c.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-16T09:58:29,808 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1722): Closing 7c7f87a6124bfe2f4049ee7803e0f98c, disabling compactions & flushes 2024-11-16T09:58:29,808 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1731751108989.7c7f87a6124bfe2f4049ee7803e0f98c. 2024-11-16T09:58:29,808 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1731751108989.7c7f87a6124bfe2f4049ee7803e0f98c. 2024-11-16T09:58:29,808 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1731751108989.7c7f87a6124bfe2f4049ee7803e0f98c. after waiting 0 ms 2024-11-16T09:58:29,808 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1731751108989.7c7f87a6124bfe2f4049ee7803e0f98c. 2024-11-16T09:58:29,808 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1731751108989.7c7f87a6124bfe2f4049ee7803e0f98c. 2024-11-16T09:58:29,808 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1676): Region close journal for 7c7f87a6124bfe2f4049ee7803e0f98c: Waiting for close lock at 1731751109808Disabling compacts and flushes for region at 1731751109808Disabling writes for close at 1731751109808Writing region close event to WAL at 1731751109808Closed at 1731751109808 2024-11-16T09:58:29,809 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=30, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ADD_TO_META 2024-11-16T09:58:29,810 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"TestAcidGuarantees,,1731751108989.7c7f87a6124bfe2f4049ee7803e0f98c.","families":{"info":[{"qualifier":"regioninfo","vlen":52,"tag":[],"timestamp":"1731751109809"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1731751109809"}]},"ts":"1731751109809"} 2024-11-16T09:58:29,811 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-11-16T09:58:29,812 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=30, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-11-16T09:58:29,812 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731751109812"}]},"ts":"1731751109812"} 2024-11-16T09:58:29,814 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLING in hbase:meta 2024-11-16T09:58:29,814 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=31, ppid=30, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=7c7f87a6124bfe2f4049ee7803e0f98c, ASSIGN}] 2024-11-16T09:58:29,815 INFO [PEWorker-2 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=31, ppid=30, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=7c7f87a6124bfe2f4049ee7803e0f98c, ASSIGN 2024-11-16T09:58:29,816 INFO [PEWorker-2 {}] assignment.TransitRegionStateProcedure(269): Starting pid=31, ppid=30, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=7c7f87a6124bfe2f4049ee7803e0f98c, ASSIGN; state=OFFLINE, location=037c58a19ddb,37267,1731751077844; forceNewPlan=false, retain=false 2024-11-16T09:58:29,967 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=31 updating hbase:meta row=7c7f87a6124bfe2f4049ee7803e0f98c, regionState=OPENING, regionLocation=037c58a19ddb,37267,1731751077844 2024-11-16T09:58:29,969 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=31, ppid=30, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=7c7f87a6124bfe2f4049ee7803e0f98c, ASSIGN because future has completed 2024-11-16T09:58:29,969 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=32, ppid=31, state=RUNNABLE, hasLock=false; OpenRegionProcedure 7c7f87a6124bfe2f4049ee7803e0f98c, server=037c58a19ddb,37267,1731751077844}] 2024-11-16T09:58:30,125 INFO [RS_OPEN_REGION-regionserver/037c58a19ddb:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] handler.AssignRegionHandler(132): Open TestAcidGuarantees,,1731751108989.7c7f87a6124bfe2f4049ee7803e0f98c. 2024-11-16T09:58:30,125 DEBUG [RS_OPEN_REGION-regionserver/037c58a19ddb:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] regionserver.HRegion(7752): Opening region: {ENCODED => 7c7f87a6124bfe2f4049ee7803e0f98c, NAME => 'TestAcidGuarantees,,1731751108989.7c7f87a6124bfe2f4049ee7803e0f98c.', STARTKEY => '', ENDKEY => ''} 2024-11-16T09:58:30,125 DEBUG [RS_OPEN_REGION-regionserver/037c58a19ddb:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestAcidGuarantees 7c7f87a6124bfe2f4049ee7803e0f98c 2024-11-16T09:58:30,125 DEBUG [RS_OPEN_REGION-regionserver/037c58a19ddb:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1731751108989.7c7f87a6124bfe2f4049ee7803e0f98c.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-16T09:58:30,126 DEBUG [RS_OPEN_REGION-regionserver/037c58a19ddb:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] regionserver.HRegion(7794): checking encryption for 7c7f87a6124bfe2f4049ee7803e0f98c 2024-11-16T09:58:30,126 DEBUG [RS_OPEN_REGION-regionserver/037c58a19ddb:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] regionserver.HRegion(7797): checking classloading for 7c7f87a6124bfe2f4049ee7803e0f98c 2024-11-16T09:58:30,127 INFO [StoreOpener-7c7f87a6124bfe2f4049ee7803e0f98c-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family A of region 7c7f87a6124bfe2f4049ee7803e0f98c 2024-11-16T09:58:30,129 INFO [StoreOpener-7c7f87a6124bfe2f4049ee7803e0f98c-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 7c7f87a6124bfe2f4049ee7803e0f98c columnFamilyName A 2024-11-16T09:58:30,129 DEBUG [StoreOpener-7c7f87a6124bfe2f4049ee7803e0f98c-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T09:58:30,130 INFO [StoreOpener-7c7f87a6124bfe2f4049ee7803e0f98c-1 {}] regionserver.HStore(327): Store=7c7f87a6124bfe2f4049ee7803e0f98c/A, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-16T09:58:30,130 INFO [StoreOpener-7c7f87a6124bfe2f4049ee7803e0f98c-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family B of region 7c7f87a6124bfe2f4049ee7803e0f98c 2024-11-16T09:58:30,131 INFO [StoreOpener-7c7f87a6124bfe2f4049ee7803e0f98c-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 7c7f87a6124bfe2f4049ee7803e0f98c columnFamilyName B 2024-11-16T09:58:30,131 DEBUG [StoreOpener-7c7f87a6124bfe2f4049ee7803e0f98c-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T09:58:30,131 INFO [StoreOpener-7c7f87a6124bfe2f4049ee7803e0f98c-1 {}] regionserver.HStore(327): Store=7c7f87a6124bfe2f4049ee7803e0f98c/B, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-16T09:58:30,131 INFO [StoreOpener-7c7f87a6124bfe2f4049ee7803e0f98c-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family C of region 7c7f87a6124bfe2f4049ee7803e0f98c 2024-11-16T09:58:30,133 INFO [StoreOpener-7c7f87a6124bfe2f4049ee7803e0f98c-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 7c7f87a6124bfe2f4049ee7803e0f98c columnFamilyName C 2024-11-16T09:58:30,133 DEBUG [StoreOpener-7c7f87a6124bfe2f4049ee7803e0f98c-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T09:58:30,133 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37877 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=30 2024-11-16T09:58:30,133 INFO [StoreOpener-7c7f87a6124bfe2f4049ee7803e0f98c-1 {}] regionserver.HStore(327): Store=7c7f87a6124bfe2f4049ee7803e0f98c/C, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-16T09:58:30,133 DEBUG [RS_OPEN_REGION-regionserver/037c58a19ddb:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] regionserver.HRegion(1038): replaying wal for 7c7f87a6124bfe2f4049ee7803e0f98c 2024-11-16T09:58:30,134 DEBUG [RS_OPEN_REGION-regionserver/037c58a19ddb:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/7c7f87a6124bfe2f4049ee7803e0f98c 2024-11-16T09:58:30,134 DEBUG [RS_OPEN_REGION-regionserver/037c58a19ddb:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/7c7f87a6124bfe2f4049ee7803e0f98c 2024-11-16T09:58:30,135 DEBUG [RS_OPEN_REGION-regionserver/037c58a19ddb:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] regionserver.HRegion(1048): stopping wal replay for 7c7f87a6124bfe2f4049ee7803e0f98c 2024-11-16T09:58:30,135 DEBUG [RS_OPEN_REGION-regionserver/037c58a19ddb:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] regionserver.HRegion(1060): Cleaning up temporary data for 7c7f87a6124bfe2f4049ee7803e0f98c 2024-11-16T09:58:30,136 DEBUG [RS_OPEN_REGION-regionserver/037c58a19ddb:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table TestAcidGuarantees descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-11-16T09:58:30,137 DEBUG [RS_OPEN_REGION-regionserver/037c58a19ddb:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] regionserver.HRegion(1093): writing seq id for 7c7f87a6124bfe2f4049ee7803e0f98c 2024-11-16T09:58:30,139 DEBUG [RS_OPEN_REGION-regionserver/037c58a19ddb:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/7c7f87a6124bfe2f4049ee7803e0f98c/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-16T09:58:30,139 INFO [RS_OPEN_REGION-regionserver/037c58a19ddb:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] regionserver.HRegion(1114): Opened 7c7f87a6124bfe2f4049ee7803e0f98c; next sequenceid=2; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=68409138, jitterRate=0.019375592470169067}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-11-16T09:58:30,139 DEBUG [RS_OPEN_REGION-regionserver/037c58a19ddb:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 7c7f87a6124bfe2f4049ee7803e0f98c 2024-11-16T09:58:30,140 DEBUG [RS_OPEN_REGION-regionserver/037c58a19ddb:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] regionserver.HRegion(1006): Region open journal for 7c7f87a6124bfe2f4049ee7803e0f98c: Running coprocessor pre-open hook at 1731751110126Writing region info on filesystem at 1731751110126Initializing all the Stores at 1731751110126Instantiating store for column family {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731751110126Instantiating store for column family {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731751110127 (+1 ms)Instantiating store for column family {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731751110127Cleaning up temporary data from old regions at 1731751110135 (+8 ms)Running coprocessor post-open hooks at 1731751110139 (+4 ms)Region opened successfully at 1731751110140 (+1 ms) 2024-11-16T09:58:30,141 INFO [RS_OPEN_REGION-regionserver/037c58a19ddb:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] regionserver.HRegionServer(2236): Post open deploy tasks for TestAcidGuarantees,,1731751108989.7c7f87a6124bfe2f4049ee7803e0f98c., pid=32, masterSystemTime=1731751110121 2024-11-16T09:58:30,143 DEBUG [RS_OPEN_REGION-regionserver/037c58a19ddb:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] regionserver.HRegionServer(2266): Finished post open deploy task for TestAcidGuarantees,,1731751108989.7c7f87a6124bfe2f4049ee7803e0f98c. 2024-11-16T09:58:30,143 INFO [RS_OPEN_REGION-regionserver/037c58a19ddb:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] handler.AssignRegionHandler(153): Opened TestAcidGuarantees,,1731751108989.7c7f87a6124bfe2f4049ee7803e0f98c. 2024-11-16T09:58:30,143 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=31 updating hbase:meta row=7c7f87a6124bfe2f4049ee7803e0f98c, regionState=OPEN, openSeqNum=2, regionLocation=037c58a19ddb,37267,1731751077844 2024-11-16T09:58:30,145 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=32, ppid=31, state=RUNNABLE, hasLock=false; OpenRegionProcedure 7c7f87a6124bfe2f4049ee7803e0f98c, server=037c58a19ddb,37267,1731751077844 because future has completed 2024-11-16T09:58:30,148 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=32, resume processing ppid=31 2024-11-16T09:58:30,148 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=32, ppid=31, state=SUCCESS, hasLock=false; OpenRegionProcedure 7c7f87a6124bfe2f4049ee7803e0f98c, server=037c58a19ddb,37267,1731751077844 in 177 msec 2024-11-16T09:58:30,151 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=31, resume processing ppid=30 2024-11-16T09:58:30,151 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=31, ppid=30, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=7c7f87a6124bfe2f4049ee7803e0f98c, ASSIGN in 334 msec 2024-11-16T09:58:30,151 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=30, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-11-16T09:58:30,152 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731751110151"}]},"ts":"1731751110151"} 2024-11-16T09:58:30,153 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLED in hbase:meta 2024-11-16T09:58:30,154 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=30, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_POST_OPERATION 2024-11-16T09:58:30,155 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=30, state=SUCCESS, hasLock=false; CreateTableProcedure table=TestAcidGuarantees in 1.1650 sec 2024-11-16T09:58:31,143 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37877 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=30 2024-11-16T09:58:31,143 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: default:TestAcidGuarantees completed 2024-11-16T09:58:31,144 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6bcb2494, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T09:58:31,144 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 037c58a19ddb,37877,-1 for getting cluster id 2024-11-16T09:58:31,145 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-16T09:58:31,146 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '1fe46ee8-cb44-44b1-8251-37103629f786' 2024-11-16T09:58:31,146 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-16T09:58:31,146 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "1fe46ee8-cb44-44b1-8251-37103629f786" 2024-11-16T09:58:31,146 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5b8d3a9c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T09:58:31,147 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [037c58a19ddb,37877,-1] 2024-11-16T09:58:31,147 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-16T09:58:31,147 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T09:58:31,148 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:45842, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-16T09:58:31,149 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1efd92d4, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T09:58:31,149 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-16T09:58:31,150 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=037c58a19ddb,37267,1731751077844, seqNum=-1] 2024-11-16T09:58:31,151 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-16T09:58:31,152 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:52390, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-16T09:58:31,153 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-11-16T09:58:31,154 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncConnectionImpl(321): The fetched master address is 037c58a19ddb,37877,1731751077131 2024-11-16T09:58:31,154 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@4ce67852 2024-11-16T09:58:31,154 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-11-16T09:58:31,155 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:45858, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-11-16T09:58:31,159 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37877 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (131072) is too small, which might cause very frequent flushing. 2024-11-16T09:58:31,160 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37877 {}] master.HMaster$14(2863): Client=jenkins//172.17.0.2 modify table TestAcidGuarantees from 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'NONE', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} to 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'NONE', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', MOB_THRESHOLD => '4', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', IS_MOB => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-16T09:58:31,166 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37877 {}] procedure2.ProcedureExecutor(1139): Stored pid=33, state=RUNNABLE:MODIFY_TABLE_PREPARE, hasLock=false; ModifyTableProcedure table=TestAcidGuarantees 2024-11-16T09:58:31,176 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37591 is added to blk_1073741968_1144 (size=995) 2024-11-16T09:58:31,578 DEBUG [PEWorker-5 {}] util.FSTableDescriptors(591): Deleted hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/.tabledesc/.tableinfo.0000000001.959 2024-11-16T09:58:31,578 INFO [PEWorker-5 {}] util.FSTableDescriptors(361): Updated tableinfo=hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/.tabledesc/.tableinfo.0000000002.995 2024-11-16T09:58:31,582 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=34, ppid=33, state=RUNNABLE:REOPEN_TABLE_REGIONS_GET_REGIONS, hasLock=false; ReopenTableRegionsProcedure table=TestAcidGuarantees}] 2024-11-16T09:58:31,588 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=35, ppid=34, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=7c7f87a6124bfe2f4049ee7803e0f98c, REOPEN/MOVE}] 2024-11-16T09:58:31,589 INFO [PEWorker-2 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=35, ppid=34, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=7c7f87a6124bfe2f4049ee7803e0f98c, REOPEN/MOVE 2024-11-16T09:58:31,590 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=35 updating hbase:meta row=7c7f87a6124bfe2f4049ee7803e0f98c, regionState=CLOSING, regionLocation=037c58a19ddb,37267,1731751077844 2024-11-16T09:58:31,591 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=35, ppid=34, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=7c7f87a6124bfe2f4049ee7803e0f98c, REOPEN/MOVE because future has completed 2024-11-16T09:58:31,592 DEBUG [PEWorker-4 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-11-16T09:58:31,592 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=36, ppid=35, state=RUNNABLE, hasLock=false; CloseRegionProcedure 7c7f87a6124bfe2f4049ee7803e0f98c, server=037c58a19ddb,37267,1731751077844}] 2024-11-16T09:58:31,744 INFO [RS_CLOSE_REGION-regionserver/037c58a19ddb:0-0 {event_type=M_RS_CLOSE_REGION, pid=36}] handler.UnassignRegionHandler(122): Close 7c7f87a6124bfe2f4049ee7803e0f98c 2024-11-16T09:58:31,744 DEBUG [RS_CLOSE_REGION-regionserver/037c58a19ddb:0-0 {event_type=M_RS_CLOSE_REGION, pid=36}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-11-16T09:58:31,744 DEBUG [RS_CLOSE_REGION-regionserver/037c58a19ddb:0-0 {event_type=M_RS_CLOSE_REGION, pid=36}] regionserver.HRegion(1722): Closing 7c7f87a6124bfe2f4049ee7803e0f98c, disabling compactions & flushes 2024-11-16T09:58:31,744 INFO [RS_CLOSE_REGION-regionserver/037c58a19ddb:0-0 {event_type=M_RS_CLOSE_REGION, pid=36}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1731751108989.7c7f87a6124bfe2f4049ee7803e0f98c. 2024-11-16T09:58:31,744 DEBUG [RS_CLOSE_REGION-regionserver/037c58a19ddb:0-0 {event_type=M_RS_CLOSE_REGION, pid=36}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1731751108989.7c7f87a6124bfe2f4049ee7803e0f98c. 2024-11-16T09:58:31,744 DEBUG [RS_CLOSE_REGION-regionserver/037c58a19ddb:0-0 {event_type=M_RS_CLOSE_REGION, pid=36}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1731751108989.7c7f87a6124bfe2f4049ee7803e0f98c. after waiting 0 ms 2024-11-16T09:58:31,744 DEBUG [RS_CLOSE_REGION-regionserver/037c58a19ddb:0-0 {event_type=M_RS_CLOSE_REGION, pid=36}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1731751108989.7c7f87a6124bfe2f4049ee7803e0f98c. 2024-11-16T09:58:31,749 DEBUG [RS_CLOSE_REGION-regionserver/037c58a19ddb:0-0 {event_type=M_RS_CLOSE_REGION, pid=36}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/7c7f87a6124bfe2f4049ee7803e0f98c/recovered.edits/4.seqid, newMaxSeqId=4, maxSeqId=1 2024-11-16T09:58:31,750 INFO [RS_CLOSE_REGION-regionserver/037c58a19ddb:0-0 {event_type=M_RS_CLOSE_REGION, pid=36}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1731751108989.7c7f87a6124bfe2f4049ee7803e0f98c. 2024-11-16T09:58:31,750 DEBUG [RS_CLOSE_REGION-regionserver/037c58a19ddb:0-0 {event_type=M_RS_CLOSE_REGION, pid=36}] regionserver.HRegion(1676): Region close journal for 7c7f87a6124bfe2f4049ee7803e0f98c: Waiting for close lock at 1731751111744Running coprocessor pre-close hooks at 1731751111744Disabling compacts and flushes for region at 1731751111744Disabling writes for close at 1731751111744Writing region close event to WAL at 1731751111745 (+1 ms)Running coprocessor post-close hooks at 1731751111749 (+4 ms)Closed at 1731751111749 2024-11-16T09:58:31,750 WARN [RS_CLOSE_REGION-regionserver/037c58a19ddb:0-0 {event_type=M_RS_CLOSE_REGION, pid=36}] regionserver.HRegionServer(3299): Not adding moved region record: 7c7f87a6124bfe2f4049ee7803e0f98c to self. 2024-11-16T09:58:31,752 INFO [RS_CLOSE_REGION-regionserver/037c58a19ddb:0-0 {event_type=M_RS_CLOSE_REGION, pid=36}] handler.UnassignRegionHandler(157): Closed 7c7f87a6124bfe2f4049ee7803e0f98c 2024-11-16T09:58:31,753 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=35 updating hbase:meta row=7c7f87a6124bfe2f4049ee7803e0f98c, regionState=CLOSED 2024-11-16T09:58:31,755 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=36, ppid=35, state=RUNNABLE, hasLock=false; CloseRegionProcedure 7c7f87a6124bfe2f4049ee7803e0f98c, server=037c58a19ddb,37267,1731751077844 because future has completed 2024-11-16T09:58:31,759 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=36, resume processing ppid=35 2024-11-16T09:58:31,759 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=36, ppid=35, state=SUCCESS, hasLock=false; CloseRegionProcedure 7c7f87a6124bfe2f4049ee7803e0f98c, server=037c58a19ddb,37267,1731751077844 in 164 msec 2024-11-16T09:58:31,759 INFO [PEWorker-2 {}] assignment.TransitRegionStateProcedure(269): Starting pid=35, ppid=34, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=7c7f87a6124bfe2f4049ee7803e0f98c, REOPEN/MOVE; state=CLOSED, location=037c58a19ddb,37267,1731751077844; forceNewPlan=false, retain=true 2024-11-16T09:58:31,910 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=35 updating hbase:meta row=7c7f87a6124bfe2f4049ee7803e0f98c, regionState=OPENING, regionLocation=037c58a19ddb,37267,1731751077844 2024-11-16T09:58:31,912 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=35, ppid=34, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=7c7f87a6124bfe2f4049ee7803e0f98c, REOPEN/MOVE because future has completed 2024-11-16T09:58:31,913 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=37, ppid=35, state=RUNNABLE, hasLock=false; OpenRegionProcedure 7c7f87a6124bfe2f4049ee7803e0f98c, server=037c58a19ddb,37267,1731751077844}] 2024-11-16T09:58:32,068 INFO [RS_OPEN_REGION-regionserver/037c58a19ddb:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] handler.AssignRegionHandler(132): Open TestAcidGuarantees,,1731751108989.7c7f87a6124bfe2f4049ee7803e0f98c. 2024-11-16T09:58:32,068 DEBUG [RS_OPEN_REGION-regionserver/037c58a19ddb:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] regionserver.HRegion(7752): Opening region: {ENCODED => 7c7f87a6124bfe2f4049ee7803e0f98c, NAME => 'TestAcidGuarantees,,1731751108989.7c7f87a6124bfe2f4049ee7803e0f98c.', STARTKEY => '', ENDKEY => ''} 2024-11-16T09:58:32,069 DEBUG [RS_OPEN_REGION-regionserver/037c58a19ddb:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestAcidGuarantees 7c7f87a6124bfe2f4049ee7803e0f98c 2024-11-16T09:58:32,069 DEBUG [RS_OPEN_REGION-regionserver/037c58a19ddb:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1731751108989.7c7f87a6124bfe2f4049ee7803e0f98c.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-16T09:58:32,069 DEBUG [RS_OPEN_REGION-regionserver/037c58a19ddb:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] regionserver.HRegion(7794): checking encryption for 7c7f87a6124bfe2f4049ee7803e0f98c 2024-11-16T09:58:32,069 DEBUG [RS_OPEN_REGION-regionserver/037c58a19ddb:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] regionserver.HRegion(7797): checking classloading for 7c7f87a6124bfe2f4049ee7803e0f98c 2024-11-16T09:58:32,073 INFO [StoreOpener-7c7f87a6124bfe2f4049ee7803e0f98c-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family A of region 7c7f87a6124bfe2f4049ee7803e0f98c 2024-11-16T09:58:32,079 INFO [StoreOpener-7c7f87a6124bfe2f4049ee7803e0f98c-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 7c7f87a6124bfe2f4049ee7803e0f98c columnFamilyName A 2024-11-16T09:58:32,081 DEBUG [StoreOpener-7c7f87a6124bfe2f4049ee7803e0f98c-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T09:58:32,081 INFO [StoreOpener-7c7f87a6124bfe2f4049ee7803e0f98c-1 {}] regionserver.HStore(327): Store=7c7f87a6124bfe2f4049ee7803e0f98c/A, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-16T09:58:32,081 INFO [StoreOpener-7c7f87a6124bfe2f4049ee7803e0f98c-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family B of region 7c7f87a6124bfe2f4049ee7803e0f98c 2024-11-16T09:58:32,082 INFO [StoreOpener-7c7f87a6124bfe2f4049ee7803e0f98c-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 7c7f87a6124bfe2f4049ee7803e0f98c columnFamilyName B 2024-11-16T09:58:32,082 DEBUG [StoreOpener-7c7f87a6124bfe2f4049ee7803e0f98c-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T09:58:32,083 INFO [StoreOpener-7c7f87a6124bfe2f4049ee7803e0f98c-1 {}] regionserver.HStore(327): Store=7c7f87a6124bfe2f4049ee7803e0f98c/B, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-16T09:58:32,083 INFO [StoreOpener-7c7f87a6124bfe2f4049ee7803e0f98c-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family C of region 7c7f87a6124bfe2f4049ee7803e0f98c 2024-11-16T09:58:32,083 INFO [StoreOpener-7c7f87a6124bfe2f4049ee7803e0f98c-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 7c7f87a6124bfe2f4049ee7803e0f98c columnFamilyName C 2024-11-16T09:58:32,083 DEBUG [StoreOpener-7c7f87a6124bfe2f4049ee7803e0f98c-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T09:58:32,084 INFO [StoreOpener-7c7f87a6124bfe2f4049ee7803e0f98c-1 {}] regionserver.HStore(327): Store=7c7f87a6124bfe2f4049ee7803e0f98c/C, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-16T09:58:32,084 DEBUG [RS_OPEN_REGION-regionserver/037c58a19ddb:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] regionserver.HRegion(1038): replaying wal for 7c7f87a6124bfe2f4049ee7803e0f98c 2024-11-16T09:58:32,085 DEBUG [RS_OPEN_REGION-regionserver/037c58a19ddb:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/7c7f87a6124bfe2f4049ee7803e0f98c 2024-11-16T09:58:32,086 DEBUG [RS_OPEN_REGION-regionserver/037c58a19ddb:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/7c7f87a6124bfe2f4049ee7803e0f98c 2024-11-16T09:58:32,086 DEBUG [RS_OPEN_REGION-regionserver/037c58a19ddb:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] regionserver.HRegion(1048): stopping wal replay for 7c7f87a6124bfe2f4049ee7803e0f98c 2024-11-16T09:58:32,087 DEBUG [RS_OPEN_REGION-regionserver/037c58a19ddb:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] regionserver.HRegion(1060): Cleaning up temporary data for 7c7f87a6124bfe2f4049ee7803e0f98c 2024-11-16T09:58:32,087 DEBUG [RS_OPEN_REGION-regionserver/037c58a19ddb:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table TestAcidGuarantees descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-11-16T09:58:32,088 DEBUG [RS_OPEN_REGION-regionserver/037c58a19ddb:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] regionserver.HRegion(1093): writing seq id for 7c7f87a6124bfe2f4049ee7803e0f98c 2024-11-16T09:58:32,089 INFO [RS_OPEN_REGION-regionserver/037c58a19ddb:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] regionserver.HRegion(1114): Opened 7c7f87a6124bfe2f4049ee7803e0f98c; next sequenceid=5; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=74888562, jitterRate=0.11592653393745422}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-11-16T09:58:32,089 DEBUG [RS_OPEN_REGION-regionserver/037c58a19ddb:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 7c7f87a6124bfe2f4049ee7803e0f98c 2024-11-16T09:58:32,090 DEBUG [RS_OPEN_REGION-regionserver/037c58a19ddb:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] regionserver.HRegion(1006): Region open journal for 7c7f87a6124bfe2f4049ee7803e0f98c: Running coprocessor pre-open hook at 1731751112069Writing region info on filesystem at 1731751112069Initializing all the Stores at 1731751112070 (+1 ms)Instantiating store for column family {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', MOB_THRESHOLD => '4', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', IS_MOB => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731751112070Instantiating store for column family {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731751112070Instantiating store for column family {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731751112070Cleaning up temporary data from old regions at 1731751112087 (+17 ms)Running coprocessor post-open hooks at 1731751112089 (+2 ms)Region opened successfully at 1731751112090 (+1 ms) 2024-11-16T09:58:32,091 INFO [RS_OPEN_REGION-regionserver/037c58a19ddb:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] regionserver.HRegionServer(2236): Post open deploy tasks for TestAcidGuarantees,,1731751108989.7c7f87a6124bfe2f4049ee7803e0f98c., pid=37, masterSystemTime=1731751112065 2024-11-16T09:58:32,093 DEBUG [RS_OPEN_REGION-regionserver/037c58a19ddb:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] regionserver.HRegionServer(2266): Finished post open deploy task for TestAcidGuarantees,,1731751108989.7c7f87a6124bfe2f4049ee7803e0f98c. 2024-11-16T09:58:32,093 INFO [RS_OPEN_REGION-regionserver/037c58a19ddb:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] handler.AssignRegionHandler(153): Opened TestAcidGuarantees,,1731751108989.7c7f87a6124bfe2f4049ee7803e0f98c. 2024-11-16T09:58:32,094 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=35 updating hbase:meta row=7c7f87a6124bfe2f4049ee7803e0f98c, regionState=OPEN, openSeqNum=5, regionLocation=037c58a19ddb,37267,1731751077844 2024-11-16T09:58:32,095 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=37, ppid=35, state=RUNNABLE, hasLock=false; OpenRegionProcedure 7c7f87a6124bfe2f4049ee7803e0f98c, server=037c58a19ddb,37267,1731751077844 because future has completed 2024-11-16T09:58:32,099 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=37, resume processing ppid=35 2024-11-16T09:58:32,099 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=37, ppid=35, state=SUCCESS, hasLock=false; OpenRegionProcedure 7c7f87a6124bfe2f4049ee7803e0f98c, server=037c58a19ddb,37267,1731751077844 in 184 msec 2024-11-16T09:58:32,101 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=35, resume processing ppid=34 2024-11-16T09:58:32,101 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=35, ppid=34, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=7c7f87a6124bfe2f4049ee7803e0f98c, REOPEN/MOVE in 511 msec 2024-11-16T09:58:32,105 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=34, resume processing ppid=33 2024-11-16T09:58:32,105 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=34, ppid=33, state=SUCCESS, hasLock=false; ReopenTableRegionsProcedure table=TestAcidGuarantees in 521 msec 2024-11-16T09:58:32,108 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=33, state=SUCCESS, hasLock=false; ModifyTableProcedure table=TestAcidGuarantees in 944 msec 2024-11-16T09:58:32,110 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37877 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=33 2024-11-16T09:58:32,110 INFO [RPCClient-NioEventLoopGroup-4-7 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: MODIFY_COLUMN_FAMILY, Table Name: default:TestAcidGuarantees completed 2024-11-16T09:58:32,111 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@68ed5d71, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T09:58:32,111 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 037c58a19ddb,37877,-1 for getting cluster id 2024-11-16T09:58:32,111 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-16T09:58:32,113 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '1fe46ee8-cb44-44b1-8251-37103629f786' 2024-11-16T09:58:32,113 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-16T09:58:32,113 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "1fe46ee8-cb44-44b1-8251-37103629f786" 2024-11-16T09:58:32,113 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5c718a86, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T09:58:32,113 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [037c58a19ddb,37877,-1] 2024-11-16T09:58:32,114 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-16T09:58:32,114 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T09:58:32,115 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:45874, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-16T09:58:32,115 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@68c8845e, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T09:58:32,116 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4406a4b8, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T09:58:32,116 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 037c58a19ddb,37877,-1 for getting cluster id 2024-11-16T09:58:32,116 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-16T09:58:32,117 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '1fe46ee8-cb44-44b1-8251-37103629f786' 2024-11-16T09:58:32,117 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-16T09:58:32,117 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "1fe46ee8-cb44-44b1-8251-37103629f786" 2024-11-16T09:58:32,117 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1e2ad28e, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T09:58:32,117 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [037c58a19ddb,37877,-1] 2024-11-16T09:58:32,118 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-16T09:58:32,118 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T09:58:32,119 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:45880, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-16T09:58:32,119 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@67ffce9d, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T09:58:32,120 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@35d0a90e, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T09:58:32,120 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 037c58a19ddb,37877,-1 for getting cluster id 2024-11-16T09:58:32,120 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-16T09:58:32,121 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '1fe46ee8-cb44-44b1-8251-37103629f786' 2024-11-16T09:58:32,121 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-16T09:58:32,121 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "1fe46ee8-cb44-44b1-8251-37103629f786" 2024-11-16T09:58:32,121 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@37505526, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T09:58:32,121 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [037c58a19ddb,37877,-1] 2024-11-16T09:58:32,121 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-16T09:58:32,122 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T09:58:32,122 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:45890, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-16T09:58:32,123 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@42c49f45, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T09:58:32,124 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@59b236c2, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T09:58:32,124 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 037c58a19ddb,37877,-1 for getting cluster id 2024-11-16T09:58:32,124 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-16T09:58:32,125 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '1fe46ee8-cb44-44b1-8251-37103629f786' 2024-11-16T09:58:32,125 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-16T09:58:32,125 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "1fe46ee8-cb44-44b1-8251-37103629f786" 2024-11-16T09:58:32,125 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@44cb8e3d, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T09:58:32,125 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [037c58a19ddb,37877,-1] 2024-11-16T09:58:32,125 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-16T09:58:32,126 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T09:58:32,126 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:45906, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-16T09:58:32,127 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@25ccef7e, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T09:58:32,128 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@26dcfede, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T09:58:32,128 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 037c58a19ddb,37877,-1 for getting cluster id 2024-11-16T09:58:32,128 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-16T09:58:32,128 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '1fe46ee8-cb44-44b1-8251-37103629f786' 2024-11-16T09:58:32,129 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-16T09:58:32,129 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "1fe46ee8-cb44-44b1-8251-37103629f786" 2024-11-16T09:58:32,129 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1ad7d0b3, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T09:58:32,129 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [037c58a19ddb,37877,-1] 2024-11-16T09:58:32,129 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-16T09:58:32,129 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T09:58:32,130 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:45926, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-16T09:58:32,131 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@47bf26bc, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T09:58:32,132 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@58ae3464, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T09:58:32,132 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 037c58a19ddb,37877,-1 for getting cluster id 2024-11-16T09:58:32,132 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-16T09:58:32,132 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '1fe46ee8-cb44-44b1-8251-37103629f786' 2024-11-16T09:58:32,133 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-16T09:58:32,133 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "1fe46ee8-cb44-44b1-8251-37103629f786" 2024-11-16T09:58:32,133 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2af88a20, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T09:58:32,133 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [037c58a19ddb,37877,-1] 2024-11-16T09:58:32,133 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-16T09:58:32,133 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T09:58:32,134 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:45942, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-16T09:58:32,135 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7efd822b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T09:58:32,136 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3bbb11d3, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T09:58:32,136 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 037c58a19ddb,37877,-1 for getting cluster id 2024-11-16T09:58:32,136 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-16T09:58:32,137 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '1fe46ee8-cb44-44b1-8251-37103629f786' 2024-11-16T09:58:32,137 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-16T09:58:32,137 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "1fe46ee8-cb44-44b1-8251-37103629f786" 2024-11-16T09:58:32,137 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5395290f, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T09:58:32,137 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [037c58a19ddb,37877,-1] 2024-11-16T09:58:32,137 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-16T09:58:32,138 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T09:58:32,138 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:45968, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-16T09:58:32,139 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3f36406c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T09:58:32,139 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@41d2a6fc, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T09:58:32,139 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 037c58a19ddb,37877,-1 for getting cluster id 2024-11-16T09:58:32,139 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-16T09:58:32,140 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '1fe46ee8-cb44-44b1-8251-37103629f786' 2024-11-16T09:58:32,140 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-16T09:58:32,140 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "1fe46ee8-cb44-44b1-8251-37103629f786" 2024-11-16T09:58:32,141 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@56fbf268, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T09:58:32,141 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [037c58a19ddb,37877,-1] 2024-11-16T09:58:32,141 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-16T09:58:32,141 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T09:58:32,142 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:45992, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-16T09:58:32,142 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2573612d, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T09:58:32,144 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@30ade04f, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T09:58:32,144 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 037c58a19ddb,37877,-1 for getting cluster id 2024-11-16T09:58:32,144 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-16T09:58:32,145 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '1fe46ee8-cb44-44b1-8251-37103629f786' 2024-11-16T09:58:32,145 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-16T09:58:32,145 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "1fe46ee8-cb44-44b1-8251-37103629f786" 2024-11-16T09:58:32,145 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@157c619b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T09:58:32,145 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [037c58a19ddb,37877,-1] 2024-11-16T09:58:32,146 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-16T09:58:32,146 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T09:58:32,147 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:46016, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-16T09:58:32,147 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@61c9840d, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T09:58:32,149 DEBUG [Thread-707 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-16T09:58:32,149 DEBUG [Thread-713 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-16T09:58:32,150 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=037c58a19ddb,37267,1731751077844, seqNum=-1] 2024-11-16T09:58:32,150 DEBUG [Thread-711 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-16T09:58:32,150 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-16T09:58:32,150 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=037c58a19ddb,37267,1731751077844, seqNum=-1] 2024-11-16T09:58:32,151 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=037c58a19ddb,37267,1731751077844, seqNum=-1] 2024-11-16T09:58:32,151 DEBUG [Thread-722 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-16T09:58:32,151 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-16T09:58:32,151 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-16T09:58:32,151 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:52392, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-16T09:58:32,152 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=037c58a19ddb,37267,1731751077844, seqNum=-1] 2024-11-16T09:58:32,152 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-16T09:58:32,152 DEBUG [Thread-724 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-16T09:58:32,153 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:52396, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-16T09:58:32,153 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:52400, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-16T09:58:32,153 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:52402, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-16T09:58:32,157 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37877 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-16T09:58:32,157 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1731751108989.7c7f87a6124bfe2f4049ee7803e0f98c., hostname=037c58a19ddb,37267,1731751077844, seqNum=5] 2024-11-16T09:58:32,159 DEBUG [Thread-724 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=037c58a19ddb,37267,1731751077844, seqNum=-1] 2024-11-16T09:58:32,159 DEBUG [Thread-724 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-16T09:58:32,159 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1731751108989.7c7f87a6124bfe2f4049ee7803e0f98c., hostname=037c58a19ddb,37267,1731751077844, seqNum=5] 2024-11-16T09:58:32,160 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1731751108989.7c7f87a6124bfe2f4049ee7803e0f98c., hostname=037c58a19ddb,37267,1731751077844, seqNum=5] 2024-11-16T09:58:32,161 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:52406, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-16T09:58:32,161 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1731751108989.7c7f87a6124bfe2f4049ee7803e0f98c., hostname=037c58a19ddb,37267,1731751077844, seqNum=5] 2024-11-16T09:58:32,162 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37877 {}] procedure2.ProcedureExecutor(1139): Stored pid=38, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=38, table=TestAcidGuarantees 2024-11-16T09:58:32,162 DEBUG [Thread-718 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-16T09:58:32,163 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37877 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=38 2024-11-16T09:58:32,164 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=037c58a19ddb,37267,1731751077844, seqNum=-1] 2024-11-16T09:58:32,164 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=38, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=38, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-16T09:58:32,164 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1731751108989.7c7f87a6124bfe2f4049ee7803e0f98c., hostname=037c58a19ddb,37267,1731751077844, seqNum=5] 2024-11-16T09:58:32,164 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-16T09:58:32,165 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:52414, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-16T09:58:32,165 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=38, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=38, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-16T09:58:32,166 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=39, ppid=38, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-16T09:58:32,170 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1731751108989.7c7f87a6124bfe2f4049ee7803e0f98c., hostname=037c58a19ddb,37267,1731751077844, seqNum=5] 2024-11-16T09:58:32,172 DEBUG [Thread-720 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-16T09:58:32,173 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=037c58a19ddb,37267,1731751077844, seqNum=-1] 2024-11-16T09:58:32,174 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-16T09:58:32,175 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:52416, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-16T09:58:32,177 DEBUG [Thread-709 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-16T09:58:32,178 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1731751108989.7c7f87a6124bfe2f4049ee7803e0f98c., hostname=037c58a19ddb,37267,1731751077844, seqNum=5] 2024-11-16T09:58:32,180 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] regionserver.HRegion(8855): Flush requested on 7c7f87a6124bfe2f4049ee7803e0f98c 2024-11-16T09:58:32,180 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-16T09:58:32,181 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 7c7f87a6124bfe2f4049ee7803e0f98c 3/3 column families, dataSize=53.67 KB heapSize=141.38 KB 2024-11-16T09:58:32,181 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=037c58a19ddb,37267,1731751077844, seqNum=-1] 2024-11-16T09:58:32,181 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-16T09:58:32,183 DEBUG [Thread-715 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-16T09:58:32,184 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=037c58a19ddb,37267,1731751077844, seqNum=-1] 2024-11-16T09:58:32,184 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:52420, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-16T09:58:32,184 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-16T09:58:32,186 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:52428, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-16T09:58:32,189 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1731751108989.7c7f87a6124bfe2f4049ee7803e0f98c., hostname=037c58a19ddb,37267,1731751077844, seqNum=5] 2024-11-16T09:58:32,191 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1731751108989.7c7f87a6124bfe2f4049ee7803e0f98c., hostname=037c58a19ddb,37267,1731751077844, seqNum=5] 2024-11-16T09:58:32,248 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241116dbe6e66079ca40bc8d84d8cc266ea18a_7c7f87a6124bfe2f4049ee7803e0f98c is 50, key is test_row_0/A:col10/1731751112180/Put/seqid=0 2024-11-16T09:58:32,258 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c7f87a6124bfe2f4049ee7803e0f98c, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T09:58:32,259 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52392 deadline: 1731751172245, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c7f87a6124bfe2f4049ee7803e0f98c, server=037c58a19ddb,37267,1731751077844 2024-11-16T09:58:32,259 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731751108989.7c7f87a6124bfe2f4049ee7803e0f98c., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 , the old value is region=TestAcidGuarantees,,1731751108989.7c7f87a6124bfe2f4049ee7803e0f98c., hostname=037c58a19ddb,37267,1731751077844, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c7f87a6124bfe2f4049ee7803e0f98c, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:58:32,259 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731751108989.7c7f87a6124bfe2f4049ee7803e0f98c., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c7f87a6124bfe2f4049ee7803e0f98c, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:58:32,260 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731751108989.7c7f87a6124bfe2f4049ee7803e0f98c., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 because the exception is null or not the one we care about 2024-11-16T09:58:32,260 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c7f87a6124bfe2f4049ee7803e0f98c, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T09:58:32,260 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52400 deadline: 1731751172251, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c7f87a6124bfe2f4049ee7803e0f98c, server=037c58a19ddb,37267,1731751077844 2024-11-16T09:58:32,260 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c7f87a6124bfe2f4049ee7803e0f98c, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T09:58:32,260 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] ipc.CallRunner(138): callId: 5 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52428 deadline: 1731751172254, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c7f87a6124bfe2f4049ee7803e0f98c, server=037c58a19ddb,37267,1731751077844 2024-11-16T09:58:32,261 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c7f87a6124bfe2f4049ee7803e0f98c, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T09:58:32,261 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] ipc.CallRunner(138): callId: 5 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52420 deadline: 1731751172256, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c7f87a6124bfe2f4049ee7803e0f98c, server=037c58a19ddb,37267,1731751077844 2024-11-16T09:58:32,261 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c7f87a6124bfe2f4049ee7803e0f98c, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T09:58:32,262 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52396 deadline: 1731751172258, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c7f87a6124bfe2f4049ee7803e0f98c, server=037c58a19ddb,37267,1731751077844 2024-11-16T09:58:32,262 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731751108989.7c7f87a6124bfe2f4049ee7803e0f98c., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 , the old value is region=TestAcidGuarantees,,1731751108989.7c7f87a6124bfe2f4049ee7803e0f98c., hostname=037c58a19ddb,37267,1731751077844, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c7f87a6124bfe2f4049ee7803e0f98c, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:58:32,262 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731751108989.7c7f87a6124bfe2f4049ee7803e0f98c., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c7f87a6124bfe2f4049ee7803e0f98c, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:58:32,262 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731751108989.7c7f87a6124bfe2f4049ee7803e0f98c., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 because the exception is null or not the one we care about 2024-11-16T09:58:32,263 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731751108989.7c7f87a6124bfe2f4049ee7803e0f98c., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 , the old value is region=TestAcidGuarantees,,1731751108989.7c7f87a6124bfe2f4049ee7803e0f98c., hostname=037c58a19ddb,37267,1731751077844, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c7f87a6124bfe2f4049ee7803e0f98c, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:58:32,263 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731751108989.7c7f87a6124bfe2f4049ee7803e0f98c., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 , the old value is region=TestAcidGuarantees,,1731751108989.7c7f87a6124bfe2f4049ee7803e0f98c., hostname=037c58a19ddb,37267,1731751077844, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c7f87a6124bfe2f4049ee7803e0f98c, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:58:32,263 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731751108989.7c7f87a6124bfe2f4049ee7803e0f98c., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 , the old value is region=TestAcidGuarantees,,1731751108989.7c7f87a6124bfe2f4049ee7803e0f98c., hostname=037c58a19ddb,37267,1731751077844, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c7f87a6124bfe2f4049ee7803e0f98c, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:58:32,263 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731751108989.7c7f87a6124bfe2f4049ee7803e0f98c., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c7f87a6124bfe2f4049ee7803e0f98c, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:58:32,263 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731751108989.7c7f87a6124bfe2f4049ee7803e0f98c., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c7f87a6124bfe2f4049ee7803e0f98c, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:58:32,263 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731751108989.7c7f87a6124bfe2f4049ee7803e0f98c., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c7f87a6124bfe2f4049ee7803e0f98c, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:58:32,263 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731751108989.7c7f87a6124bfe2f4049ee7803e0f98c., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 because the exception is null or not the one we care about 2024-11-16T09:58:32,263 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731751108989.7c7f87a6124bfe2f4049ee7803e0f98c., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 because the exception is null or not the one we care about 2024-11-16T09:58:32,263 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731751108989.7c7f87a6124bfe2f4049ee7803e0f98c., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 because the exception is null or not the one we care about 2024-11-16T09:58:32,273 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37877 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=38 2024-11-16T09:58:32,276 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37591 is added to blk_1073741969_1145 (size=14594) 2024-11-16T09:58:32,277 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T09:58:32,284 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241116dbe6e66079ca40bc8d84d8cc266ea18a_7c7f87a6124bfe2f4049ee7803e0f98c to hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241116dbe6e66079ca40bc8d84d8cc266ea18a_7c7f87a6124bfe2f4049ee7803e0f98c 2024-11-16T09:58:32,288 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/7c7f87a6124bfe2f4049ee7803e0f98c/.tmp/A/c6a8bb2378f148b9ad055005e5046ab4, store: [table=TestAcidGuarantees family=A region=7c7f87a6124bfe2f4049ee7803e0f98c] 2024-11-16T09:58:32,296 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/7c7f87a6124bfe2f4049ee7803e0f98c/.tmp/A/c6a8bb2378f148b9ad055005e5046ab4 is 175, key is test_row_0/A:col10/1731751112180/Put/seqid=0 2024-11-16T09:58:32,320 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=37267 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=39 2024-11-16T09:58:32,320 DEBUG [RS_FLUSH_OPERATIONS-regionserver/037c58a19ddb:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731751108989.7c7f87a6124bfe2f4049ee7803e0f98c. 2024-11-16T09:58:32,320 DEBUG [RS_FLUSH_OPERATIONS-regionserver/037c58a19ddb:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731751108989.7c7f87a6124bfe2f4049ee7803e0f98c. as already flushing 2024-11-16T09:58:32,320 DEBUG [RS_FLUSH_OPERATIONS-regionserver/037c58a19ddb:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731751108989.7c7f87a6124bfe2f4049ee7803e0f98c. 2024-11-16T09:58:32,320 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37591 is added to blk_1073741970_1146 (size=39549) 2024-11-16T09:58:32,320 ERROR [RS_FLUSH_OPERATIONS-regionserver/037c58a19ddb:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] handler.RSProcedureHandler(58): pid=39 java.io.IOException: Unable to complete flush {ENCODED => 7c7f87a6124bfe2f4049ee7803e0f98c, NAME => 'TestAcidGuarantees,,1731751108989.7c7f87a6124bfe2f4049ee7803e0f98c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T09:58:32,320 DEBUG [RS_FLUSH_OPERATIONS-regionserver/037c58a19ddb:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=39 java.io.IOException: Unable to complete flush {ENCODED => 7c7f87a6124bfe2f4049ee7803e0f98c, NAME => 'TestAcidGuarantees,,1731751108989.7c7f87a6124bfe2f4049ee7803e0f98c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T09:58:32,322 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37877 {}] master.HMaster(4177): Remote procedure failed, pid=39 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 7c7f87a6124bfe2f4049ee7803e0f98c, NAME => 'TestAcidGuarantees,,1731751108989.7c7f87a6124bfe2f4049ee7803e0f98c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 7c7f87a6124bfe2f4049ee7803e0f98c, NAME => 'TestAcidGuarantees,,1731751108989.7c7f87a6124bfe2f4049ee7803e0f98c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T09:58:32,366 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c7f87a6124bfe2f4049ee7803e0f98c, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T09:58:32,367 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52392 deadline: 1731751172364, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c7f87a6124bfe2f4049ee7803e0f98c, server=037c58a19ddb,37267,1731751077844 2024-11-16T09:58:32,367 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731751108989.7c7f87a6124bfe2f4049ee7803e0f98c., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 , the old value is region=TestAcidGuarantees,,1731751108989.7c7f87a6124bfe2f4049ee7803e0f98c., hostname=037c58a19ddb,37267,1731751077844, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c7f87a6124bfe2f4049ee7803e0f98c, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:58:32,367 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731751108989.7c7f87a6124bfe2f4049ee7803e0f98c., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c7f87a6124bfe2f4049ee7803e0f98c, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:58:32,367 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731751108989.7c7f87a6124bfe2f4049ee7803e0f98c., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 because the exception is null or not the one we care about 2024-11-16T09:58:32,378 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c7f87a6124bfe2f4049ee7803e0f98c, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T09:58:32,378 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52396 deadline: 1731751172373, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c7f87a6124bfe2f4049ee7803e0f98c, server=037c58a19ddb,37267,1731751077844 2024-11-16T09:58:32,379 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c7f87a6124bfe2f4049ee7803e0f98c, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T09:58:32,379 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52400 deadline: 1731751172374, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c7f87a6124bfe2f4049ee7803e0f98c, server=037c58a19ddb,37267,1731751077844 2024-11-16T09:58:32,379 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c7f87a6124bfe2f4049ee7803e0f98c, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T09:58:32,379 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] ipc.CallRunner(138): callId: 6 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52420 deadline: 1731751172374, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c7f87a6124bfe2f4049ee7803e0f98c, server=037c58a19ddb,37267,1731751077844 2024-11-16T09:58:32,379 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731751108989.7c7f87a6124bfe2f4049ee7803e0f98c., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 , the old value is region=TestAcidGuarantees,,1731751108989.7c7f87a6124bfe2f4049ee7803e0f98c., hostname=037c58a19ddb,37267,1731751077844, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c7f87a6124bfe2f4049ee7803e0f98c, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:58:32,379 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731751108989.7c7f87a6124bfe2f4049ee7803e0f98c., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c7f87a6124bfe2f4049ee7803e0f98c, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:58:32,380 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731751108989.7c7f87a6124bfe2f4049ee7803e0f98c., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 because the exception is null or not the one we care about 2024-11-16T09:58:32,380 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c7f87a6124bfe2f4049ee7803e0f98c, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T09:58:32,380 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] ipc.CallRunner(138): callId: 6 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52428 deadline: 1731751172374, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c7f87a6124bfe2f4049ee7803e0f98c, server=037c58a19ddb,37267,1731751077844 2024-11-16T09:58:32,380 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731751108989.7c7f87a6124bfe2f4049ee7803e0f98c., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 , the old value is region=TestAcidGuarantees,,1731751108989.7c7f87a6124bfe2f4049ee7803e0f98c., hostname=037c58a19ddb,37267,1731751077844, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c7f87a6124bfe2f4049ee7803e0f98c, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:58:32,380 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731751108989.7c7f87a6124bfe2f4049ee7803e0f98c., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c7f87a6124bfe2f4049ee7803e0f98c, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:58:32,380 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731751108989.7c7f87a6124bfe2f4049ee7803e0f98c., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 because the exception is null or not the one we care about 2024-11-16T09:58:32,380 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731751108989.7c7f87a6124bfe2f4049ee7803e0f98c., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 , the old value is region=TestAcidGuarantees,,1731751108989.7c7f87a6124bfe2f4049ee7803e0f98c., hostname=037c58a19ddb,37267,1731751077844, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c7f87a6124bfe2f4049ee7803e0f98c, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:58:32,380 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731751108989.7c7f87a6124bfe2f4049ee7803e0f98c., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c7f87a6124bfe2f4049ee7803e0f98c, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:58:32,381 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731751108989.7c7f87a6124bfe2f4049ee7803e0f98c., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 because the exception is null or not the one we care about 2024-11-16T09:58:32,381 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731751108989.7c7f87a6124bfe2f4049ee7803e0f98c., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 , the old value is region=TestAcidGuarantees,,1731751108989.7c7f87a6124bfe2f4049ee7803e0f98c., hostname=037c58a19ddb,37267,1731751077844, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c7f87a6124bfe2f4049ee7803e0f98c, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:58:32,381 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731751108989.7c7f87a6124bfe2f4049ee7803e0f98c., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c7f87a6124bfe2f4049ee7803e0f98c, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:58:32,381 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731751108989.7c7f87a6124bfe2f4049ee7803e0f98c., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 because the exception is null or not the one we care about 2024-11-16T09:58:32,474 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=37267 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=39 2024-11-16T09:58:32,474 DEBUG [RS_FLUSH_OPERATIONS-regionserver/037c58a19ddb:0-2 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731751108989.7c7f87a6124bfe2f4049ee7803e0f98c. 2024-11-16T09:58:32,474 DEBUG [RS_FLUSH_OPERATIONS-regionserver/037c58a19ddb:0-2 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731751108989.7c7f87a6124bfe2f4049ee7803e0f98c. as already flushing 2024-11-16T09:58:32,474 DEBUG [RS_FLUSH_OPERATIONS-regionserver/037c58a19ddb:0-2 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731751108989.7c7f87a6124bfe2f4049ee7803e0f98c. 2024-11-16T09:58:32,474 ERROR [RS_FLUSH_OPERATIONS-regionserver/037c58a19ddb:0-2 {event_type=RS_FLUSH_REGIONS, pid=39}] handler.RSProcedureHandler(58): pid=39 java.io.IOException: Unable to complete flush {ENCODED => 7c7f87a6124bfe2f4049ee7803e0f98c, NAME => 'TestAcidGuarantees,,1731751108989.7c7f87a6124bfe2f4049ee7803e0f98c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T09:58:32,475 DEBUG [RS_FLUSH_OPERATIONS-regionserver/037c58a19ddb:0-2 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=39 java.io.IOException: Unable to complete flush {ENCODED => 7c7f87a6124bfe2f4049ee7803e0f98c, NAME => 'TestAcidGuarantees,,1731751108989.7c7f87a6124bfe2f4049ee7803e0f98c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T09:58:32,476 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37877 {}] master.HMaster(4177): Remote procedure failed, pid=39 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 7c7f87a6124bfe2f4049ee7803e0f98c, NAME => 'TestAcidGuarantees,,1731751108989.7c7f87a6124bfe2f4049ee7803e0f98c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 7c7f87a6124bfe2f4049ee7803e0f98c, NAME => 'TestAcidGuarantees,,1731751108989.7c7f87a6124bfe2f4049ee7803e0f98c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T09:58:32,483 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37877 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=38 2024-11-16T09:58:32,575 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c7f87a6124bfe2f4049ee7803e0f98c, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T09:58:32,575 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52392 deadline: 1731751172573, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c7f87a6124bfe2f4049ee7803e0f98c, server=037c58a19ddb,37267,1731751077844 2024-11-16T09:58:32,575 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731751108989.7c7f87a6124bfe2f4049ee7803e0f98c., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 , the old value is region=TestAcidGuarantees,,1731751108989.7c7f87a6124bfe2f4049ee7803e0f98c., hostname=037c58a19ddb,37267,1731751077844, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c7f87a6124bfe2f4049ee7803e0f98c, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:58:32,575 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731751108989.7c7f87a6124bfe2f4049ee7803e0f98c., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c7f87a6124bfe2f4049ee7803e0f98c, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:58:32,576 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731751108989.7c7f87a6124bfe2f4049ee7803e0f98c., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 because the exception is null or not the one we care about 2024-11-16T09:58:32,586 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c7f87a6124bfe2f4049ee7803e0f98c, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T09:58:32,586 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52400 deadline: 1731751172584, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c7f87a6124bfe2f4049ee7803e0f98c, server=037c58a19ddb,37267,1731751077844 2024-11-16T09:58:32,587 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731751108989.7c7f87a6124bfe2f4049ee7803e0f98c., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 , the old value is region=TestAcidGuarantees,,1731751108989.7c7f87a6124bfe2f4049ee7803e0f98c., hostname=037c58a19ddb,37267,1731751077844, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c7f87a6124bfe2f4049ee7803e0f98c, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:58:32,587 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731751108989.7c7f87a6124bfe2f4049ee7803e0f98c., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c7f87a6124bfe2f4049ee7803e0f98c, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:58:32,587 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731751108989.7c7f87a6124bfe2f4049ee7803e0f98c., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 because the exception is null or not the one we care about 2024-11-16T09:58:32,587 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c7f87a6124bfe2f4049ee7803e0f98c, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T09:58:32,587 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52428 deadline: 1731751172584, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c7f87a6124bfe2f4049ee7803e0f98c, server=037c58a19ddb,37267,1731751077844 2024-11-16T09:58:32,587 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731751108989.7c7f87a6124bfe2f4049ee7803e0f98c., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 , the old value is region=TestAcidGuarantees,,1731751108989.7c7f87a6124bfe2f4049ee7803e0f98c., hostname=037c58a19ddb,37267,1731751077844, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c7f87a6124bfe2f4049ee7803e0f98c, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:58:32,587 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731751108989.7c7f87a6124bfe2f4049ee7803e0f98c., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c7f87a6124bfe2f4049ee7803e0f98c, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:58:32,587 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731751108989.7c7f87a6124bfe2f4049ee7803e0f98c., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 because the exception is null or not the one we care about 2024-11-16T09:58:32,587 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c7f87a6124bfe2f4049ee7803e0f98c, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T09:58:32,588 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52396 deadline: 1731751172584, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c7f87a6124bfe2f4049ee7803e0f98c, server=037c58a19ddb,37267,1731751077844 2024-11-16T09:58:32,588 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731751108989.7c7f87a6124bfe2f4049ee7803e0f98c., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 , the old value is region=TestAcidGuarantees,,1731751108989.7c7f87a6124bfe2f4049ee7803e0f98c., hostname=037c58a19ddb,37267,1731751077844, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c7f87a6124bfe2f4049ee7803e0f98c, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:58:32,588 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731751108989.7c7f87a6124bfe2f4049ee7803e0f98c., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c7f87a6124bfe2f4049ee7803e0f98c, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:58:32,588 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c7f87a6124bfe2f4049ee7803e0f98c, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T09:58:32,588 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731751108989.7c7f87a6124bfe2f4049ee7803e0f98c., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 because the exception is null or not the one we care about 2024-11-16T09:58:32,588 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52420 deadline: 1731751172585, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c7f87a6124bfe2f4049ee7803e0f98c, server=037c58a19ddb,37267,1731751077844 2024-11-16T09:58:32,589 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731751108989.7c7f87a6124bfe2f4049ee7803e0f98c., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 , the old value is region=TestAcidGuarantees,,1731751108989.7c7f87a6124bfe2f4049ee7803e0f98c., hostname=037c58a19ddb,37267,1731751077844, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c7f87a6124bfe2f4049ee7803e0f98c, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:58:32,589 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731751108989.7c7f87a6124bfe2f4049ee7803e0f98c., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c7f87a6124bfe2f4049ee7803e0f98c, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:58:32,589 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731751108989.7c7f87a6124bfe2f4049ee7803e0f98c., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 because the exception is null or not the one we care about 2024-11-16T09:58:32,628 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=37267 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=39 2024-11-16T09:58:32,629 DEBUG [RS_FLUSH_OPERATIONS-regionserver/037c58a19ddb:0-0 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731751108989.7c7f87a6124bfe2f4049ee7803e0f98c. 2024-11-16T09:58:32,629 DEBUG [RS_FLUSH_OPERATIONS-regionserver/037c58a19ddb:0-0 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731751108989.7c7f87a6124bfe2f4049ee7803e0f98c. as already flushing 2024-11-16T09:58:32,629 DEBUG [RS_FLUSH_OPERATIONS-regionserver/037c58a19ddb:0-0 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731751108989.7c7f87a6124bfe2f4049ee7803e0f98c. 2024-11-16T09:58:32,629 ERROR [RS_FLUSH_OPERATIONS-regionserver/037c58a19ddb:0-0 {event_type=RS_FLUSH_REGIONS, pid=39}] handler.RSProcedureHandler(58): pid=39 java.io.IOException: Unable to complete flush {ENCODED => 7c7f87a6124bfe2f4049ee7803e0f98c, NAME => 'TestAcidGuarantees,,1731751108989.7c7f87a6124bfe2f4049ee7803e0f98c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T09:58:32,629 DEBUG [RS_FLUSH_OPERATIONS-regionserver/037c58a19ddb:0-0 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=39 java.io.IOException: Unable to complete flush {ENCODED => 7c7f87a6124bfe2f4049ee7803e0f98c, NAME => 'TestAcidGuarantees,,1731751108989.7c7f87a6124bfe2f4049ee7803e0f98c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T09:58:32,630 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37877 {}] master.HMaster(4177): Remote procedure failed, pid=39 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 7c7f87a6124bfe2f4049ee7803e0f98c, NAME => 'TestAcidGuarantees,,1731751108989.7c7f87a6124bfe2f4049ee7803e0f98c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 7c7f87a6124bfe2f4049ee7803e0f98c, NAME => 'TestAcidGuarantees,,1731751108989.7c7f87a6124bfe2f4049ee7803e0f98c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T09:58:32,726 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=17, memsize=22.4 K, hasBloomFilter=true, into tmp file hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/7c7f87a6124bfe2f4049ee7803e0f98c/.tmp/A/c6a8bb2378f148b9ad055005e5046ab4 2024-11-16T09:58:32,774 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/7c7f87a6124bfe2f4049ee7803e0f98c/.tmp/B/bf8668dd1fd7427caec8099ddc5334f7 is 50, key is test_row_0/B:col10/1731751112180/Put/seqid=0 2024-11-16T09:58:32,782 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=37267 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=39 2024-11-16T09:58:32,782 DEBUG [RS_FLUSH_OPERATIONS-regionserver/037c58a19ddb:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731751108989.7c7f87a6124bfe2f4049ee7803e0f98c. 2024-11-16T09:58:32,782 DEBUG [RS_FLUSH_OPERATIONS-regionserver/037c58a19ddb:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731751108989.7c7f87a6124bfe2f4049ee7803e0f98c. as already flushing 2024-11-16T09:58:32,783 DEBUG [RS_FLUSH_OPERATIONS-regionserver/037c58a19ddb:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731751108989.7c7f87a6124bfe2f4049ee7803e0f98c. 2024-11-16T09:58:32,783 ERROR [RS_FLUSH_OPERATIONS-regionserver/037c58a19ddb:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] handler.RSProcedureHandler(58): pid=39 java.io.IOException: Unable to complete flush {ENCODED => 7c7f87a6124bfe2f4049ee7803e0f98c, NAME => 'TestAcidGuarantees,,1731751108989.7c7f87a6124bfe2f4049ee7803e0f98c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T09:58:32,783 DEBUG [RS_FLUSH_OPERATIONS-regionserver/037c58a19ddb:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=39 java.io.IOException: Unable to complete flush {ENCODED => 7c7f87a6124bfe2f4049ee7803e0f98c, NAME => 'TestAcidGuarantees,,1731751108989.7c7f87a6124bfe2f4049ee7803e0f98c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T09:58:32,784 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37877 {}] master.HMaster(4177): Remote procedure failed, pid=39 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 7c7f87a6124bfe2f4049ee7803e0f98c, NAME => 'TestAcidGuarantees,,1731751108989.7c7f87a6124bfe2f4049ee7803e0f98c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 7c7f87a6124bfe2f4049ee7803e0f98c, NAME => 'TestAcidGuarantees,,1731751108989.7c7f87a6124bfe2f4049ee7803e0f98c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T09:58:32,793 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37877 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=38 2024-11-16T09:58:32,804 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37591 is added to blk_1073741971_1147 (size=12001) 2024-11-16T09:58:32,809 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=17 (bloomFilter=true), to=hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/7c7f87a6124bfe2f4049ee7803e0f98c/.tmp/B/bf8668dd1fd7427caec8099ddc5334f7 2024-11-16T09:58:32,856 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/7c7f87a6124bfe2f4049ee7803e0f98c/.tmp/C/ccfea16deb0641a3ab89085b418edb9b is 50, key is test_row_0/C:col10/1731751112180/Put/seqid=0 2024-11-16T09:58:32,876 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37591 is added to blk_1073741972_1148 (size=12001) 2024-11-16T09:58:32,888 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c7f87a6124bfe2f4049ee7803e0f98c, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T09:58:32,888 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52392 deadline: 1731751172883, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c7f87a6124bfe2f4049ee7803e0f98c, server=037c58a19ddb,37267,1731751077844 2024-11-16T09:58:32,889 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731751108989.7c7f87a6124bfe2f4049ee7803e0f98c., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 , the old value is region=TestAcidGuarantees,,1731751108989.7c7f87a6124bfe2f4049ee7803e0f98c., hostname=037c58a19ddb,37267,1731751077844, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c7f87a6124bfe2f4049ee7803e0f98c, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:58:32,889 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731751108989.7c7f87a6124bfe2f4049ee7803e0f98c., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c7f87a6124bfe2f4049ee7803e0f98c, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:58:32,889 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731751108989.7c7f87a6124bfe2f4049ee7803e0f98c., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 because the exception is null or not the one we care about 2024-11-16T09:58:32,898 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c7f87a6124bfe2f4049ee7803e0f98c, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T09:58:32,898 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52396 deadline: 1731751172894, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c7f87a6124bfe2f4049ee7803e0f98c, server=037c58a19ddb,37267,1731751077844 2024-11-16T09:58:32,898 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c7f87a6124bfe2f4049ee7803e0f98c, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T09:58:32,898 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52420 deadline: 1731751172895, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c7f87a6124bfe2f4049ee7803e0f98c, server=037c58a19ddb,37267,1731751077844 2024-11-16T09:58:32,899 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c7f87a6124bfe2f4049ee7803e0f98c, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T09:58:32,899 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52400 deadline: 1731751172895, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c7f87a6124bfe2f4049ee7803e0f98c, server=037c58a19ddb,37267,1731751077844 2024-11-16T09:58:32,899 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731751108989.7c7f87a6124bfe2f4049ee7803e0f98c., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 , the old value is region=TestAcidGuarantees,,1731751108989.7c7f87a6124bfe2f4049ee7803e0f98c., hostname=037c58a19ddb,37267,1731751077844, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c7f87a6124bfe2f4049ee7803e0f98c, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:58:32,899 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731751108989.7c7f87a6124bfe2f4049ee7803e0f98c., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c7f87a6124bfe2f4049ee7803e0f98c, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:58:32,899 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731751108989.7c7f87a6124bfe2f4049ee7803e0f98c., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 because the exception is null or not the one we care about 2024-11-16T09:58:32,899 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731751108989.7c7f87a6124bfe2f4049ee7803e0f98c., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 , the old value is region=TestAcidGuarantees,,1731751108989.7c7f87a6124bfe2f4049ee7803e0f98c., hostname=037c58a19ddb,37267,1731751077844, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c7f87a6124bfe2f4049ee7803e0f98c, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:58:32,899 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c7f87a6124bfe2f4049ee7803e0f98c, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T09:58:32,899 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731751108989.7c7f87a6124bfe2f4049ee7803e0f98c., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c7f87a6124bfe2f4049ee7803e0f98c, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:58:32,899 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731751108989.7c7f87a6124bfe2f4049ee7803e0f98c., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 because the exception is null or not the one we care about 2024-11-16T09:58:32,899 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52428 deadline: 1731751172895, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c7f87a6124bfe2f4049ee7803e0f98c, server=037c58a19ddb,37267,1731751077844 2024-11-16T09:58:32,900 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731751108989.7c7f87a6124bfe2f4049ee7803e0f98c., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 , the old value is region=TestAcidGuarantees,,1731751108989.7c7f87a6124bfe2f4049ee7803e0f98c., hostname=037c58a19ddb,37267,1731751077844, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c7f87a6124bfe2f4049ee7803e0f98c, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:58:32,900 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731751108989.7c7f87a6124bfe2f4049ee7803e0f98c., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 , the old value is region=TestAcidGuarantees,,1731751108989.7c7f87a6124bfe2f4049ee7803e0f98c., hostname=037c58a19ddb,37267,1731751077844, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c7f87a6124bfe2f4049ee7803e0f98c, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:58:32,900 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731751108989.7c7f87a6124bfe2f4049ee7803e0f98c., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c7f87a6124bfe2f4049ee7803e0f98c, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:58:32,900 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731751108989.7c7f87a6124bfe2f4049ee7803e0f98c., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c7f87a6124bfe2f4049ee7803e0f98c, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:58:32,900 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731751108989.7c7f87a6124bfe2f4049ee7803e0f98c., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 because the exception is null or not the one we care about 2024-11-16T09:58:32,900 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731751108989.7c7f87a6124bfe2f4049ee7803e0f98c., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 because the exception is null or not the one we care about 2024-11-16T09:58:32,937 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=37267 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=39 2024-11-16T09:58:32,937 DEBUG [RS_FLUSH_OPERATIONS-regionserver/037c58a19ddb:0-2 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731751108989.7c7f87a6124bfe2f4049ee7803e0f98c. 2024-11-16T09:58:32,938 DEBUG [RS_FLUSH_OPERATIONS-regionserver/037c58a19ddb:0-2 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731751108989.7c7f87a6124bfe2f4049ee7803e0f98c. as already flushing 2024-11-16T09:58:32,938 DEBUG [RS_FLUSH_OPERATIONS-regionserver/037c58a19ddb:0-2 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731751108989.7c7f87a6124bfe2f4049ee7803e0f98c. 2024-11-16T09:58:32,938 ERROR [RS_FLUSH_OPERATIONS-regionserver/037c58a19ddb:0-2 {event_type=RS_FLUSH_REGIONS, pid=39}] handler.RSProcedureHandler(58): pid=39 java.io.IOException: Unable to complete flush {ENCODED => 7c7f87a6124bfe2f4049ee7803e0f98c, NAME => 'TestAcidGuarantees,,1731751108989.7c7f87a6124bfe2f4049ee7803e0f98c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T09:58:32,938 DEBUG [RS_FLUSH_OPERATIONS-regionserver/037c58a19ddb:0-2 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=39 java.io.IOException: Unable to complete flush {ENCODED => 7c7f87a6124bfe2f4049ee7803e0f98c, NAME => 'TestAcidGuarantees,,1731751108989.7c7f87a6124bfe2f4049ee7803e0f98c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T09:58:32,939 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37877 {}] master.HMaster(4177): Remote procedure failed, pid=39 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 7c7f87a6124bfe2f4049ee7803e0f98c, NAME => 'TestAcidGuarantees,,1731751108989.7c7f87a6124bfe2f4049ee7803e0f98c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 7c7f87a6124bfe2f4049ee7803e0f98c, NAME => 'TestAcidGuarantees,,1731751108989.7c7f87a6124bfe2f4049ee7803e0f98c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T09:58:33,093 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=37267 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=39 2024-11-16T09:58:33,093 DEBUG [RS_FLUSH_OPERATIONS-regionserver/037c58a19ddb:0-0 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731751108989.7c7f87a6124bfe2f4049ee7803e0f98c. 2024-11-16T09:58:33,094 DEBUG [RS_FLUSH_OPERATIONS-regionserver/037c58a19ddb:0-0 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731751108989.7c7f87a6124bfe2f4049ee7803e0f98c. as already flushing 2024-11-16T09:58:33,094 DEBUG [RS_FLUSH_OPERATIONS-regionserver/037c58a19ddb:0-0 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731751108989.7c7f87a6124bfe2f4049ee7803e0f98c. 2024-11-16T09:58:33,094 ERROR [RS_FLUSH_OPERATIONS-regionserver/037c58a19ddb:0-0 {event_type=RS_FLUSH_REGIONS, pid=39}] handler.RSProcedureHandler(58): pid=39 java.io.IOException: Unable to complete flush {ENCODED => 7c7f87a6124bfe2f4049ee7803e0f98c, NAME => 'TestAcidGuarantees,,1731751108989.7c7f87a6124bfe2f4049ee7803e0f98c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T09:58:33,094 DEBUG [RS_FLUSH_OPERATIONS-regionserver/037c58a19ddb:0-0 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=39 java.io.IOException: Unable to complete flush {ENCODED => 7c7f87a6124bfe2f4049ee7803e0f98c, NAME => 'TestAcidGuarantees,,1731751108989.7c7f87a6124bfe2f4049ee7803e0f98c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T09:58:33,097 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37877 {}] master.HMaster(4177): Remote procedure failed, pid=39 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 7c7f87a6124bfe2f4049ee7803e0f98c, NAME => 'TestAcidGuarantees,,1731751108989.7c7f87a6124bfe2f4049ee7803e0f98c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 7c7f87a6124bfe2f4049ee7803e0f98c, NAME => 'TestAcidGuarantees,,1731751108989.7c7f87a6124bfe2f4049ee7803e0f98c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T09:58:33,249 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=37267 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=39 2024-11-16T09:58:33,249 DEBUG [RS_FLUSH_OPERATIONS-regionserver/037c58a19ddb:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731751108989.7c7f87a6124bfe2f4049ee7803e0f98c. 2024-11-16T09:58:33,250 DEBUG [RS_FLUSH_OPERATIONS-regionserver/037c58a19ddb:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731751108989.7c7f87a6124bfe2f4049ee7803e0f98c. as already flushing 2024-11-16T09:58:33,250 DEBUG [RS_FLUSH_OPERATIONS-regionserver/037c58a19ddb:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731751108989.7c7f87a6124bfe2f4049ee7803e0f98c. 2024-11-16T09:58:33,250 ERROR [RS_FLUSH_OPERATIONS-regionserver/037c58a19ddb:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] handler.RSProcedureHandler(58): pid=39 java.io.IOException: Unable to complete flush {ENCODED => 7c7f87a6124bfe2f4049ee7803e0f98c, NAME => 'TestAcidGuarantees,,1731751108989.7c7f87a6124bfe2f4049ee7803e0f98c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T09:58:33,250 DEBUG [RS_FLUSH_OPERATIONS-regionserver/037c58a19ddb:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=39 java.io.IOException: Unable to complete flush {ENCODED => 7c7f87a6124bfe2f4049ee7803e0f98c, NAME => 'TestAcidGuarantees,,1731751108989.7c7f87a6124bfe2f4049ee7803e0f98c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T09:58:33,251 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37877 {}] master.HMaster(4177): Remote procedure failed, pid=39 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 7c7f87a6124bfe2f4049ee7803e0f98c, NAME => 'TestAcidGuarantees,,1731751108989.7c7f87a6124bfe2f4049ee7803e0f98c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 7c7f87a6124bfe2f4049ee7803e0f98c, NAME => 'TestAcidGuarantees,,1731751108989.7c7f87a6124bfe2f4049ee7803e0f98c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T09:58:33,278 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=17 (bloomFilter=true), to=hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/7c7f87a6124bfe2f4049ee7803e0f98c/.tmp/C/ccfea16deb0641a3ab89085b418edb9b 2024-11-16T09:58:33,285 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/7c7f87a6124bfe2f4049ee7803e0f98c/.tmp/A/c6a8bb2378f148b9ad055005e5046ab4 as hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/7c7f87a6124bfe2f4049ee7803e0f98c/A/c6a8bb2378f148b9ad055005e5046ab4 2024-11-16T09:58:33,291 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/7c7f87a6124bfe2f4049ee7803e0f98c/A/c6a8bb2378f148b9ad055005e5046ab4, entries=200, sequenceid=17, filesize=38.6 K 2024-11-16T09:58:33,292 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/7c7f87a6124bfe2f4049ee7803e0f98c/.tmp/B/bf8668dd1fd7427caec8099ddc5334f7 as hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/7c7f87a6124bfe2f4049ee7803e0f98c/B/bf8668dd1fd7427caec8099ddc5334f7 2024-11-16T09:58:33,303 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37877 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=38 2024-11-16T09:58:33,311 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/7c7f87a6124bfe2f4049ee7803e0f98c/B/bf8668dd1fd7427caec8099ddc5334f7, entries=150, sequenceid=17, filesize=11.7 K 2024-11-16T09:58:33,320 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/7c7f87a6124bfe2f4049ee7803e0f98c/.tmp/C/ccfea16deb0641a3ab89085b418edb9b as hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/7c7f87a6124bfe2f4049ee7803e0f98c/C/ccfea16deb0641a3ab89085b418edb9b 2024-11-16T09:58:33,335 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/7c7f87a6124bfe2f4049ee7803e0f98c/C/ccfea16deb0641a3ab89085b418edb9b, entries=150, sequenceid=17, filesize=11.7 K 2024-11-16T09:58:33,336 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~67.09 KB/68700, heapSize ~176.48 KB/180720, currentSize=134.18 KB/137400 for 7c7f87a6124bfe2f4049ee7803e0f98c in 1156ms, sequenceid=17, compaction requested=false 2024-11-16T09:58:33,336 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 7c7f87a6124bfe2f4049ee7803e0f98c: 2024-11-16T09:58:33,397 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] regionserver.HRegion(8855): Flush requested on 7c7f87a6124bfe2f4049ee7803e0f98c 2024-11-16T09:58:33,397 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-16T09:58:33,397 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 7c7f87a6124bfe2f4049ee7803e0f98c 3/3 column families, dataSize=140.89 KB heapSize=369.89 KB 2024-11-16T09:58:33,406 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=37267 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=39 2024-11-16T09:58:33,406 DEBUG [RS_FLUSH_OPERATIONS-regionserver/037c58a19ddb:0-2 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731751108989.7c7f87a6124bfe2f4049ee7803e0f98c. 2024-11-16T09:58:33,406 DEBUG [RS_FLUSH_OPERATIONS-regionserver/037c58a19ddb:0-2 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731751108989.7c7f87a6124bfe2f4049ee7803e0f98c. as already flushing 2024-11-16T09:58:33,406 DEBUG [RS_FLUSH_OPERATIONS-regionserver/037c58a19ddb:0-2 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731751108989.7c7f87a6124bfe2f4049ee7803e0f98c. 2024-11-16T09:58:33,407 ERROR [RS_FLUSH_OPERATIONS-regionserver/037c58a19ddb:0-2 {event_type=RS_FLUSH_REGIONS, pid=39}] handler.RSProcedureHandler(58): pid=39 java.io.IOException: Unable to complete flush {ENCODED => 7c7f87a6124bfe2f4049ee7803e0f98c, NAME => 'TestAcidGuarantees,,1731751108989.7c7f87a6124bfe2f4049ee7803e0f98c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T09:58:33,407 DEBUG [RS_FLUSH_OPERATIONS-regionserver/037c58a19ddb:0-2 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=39 java.io.IOException: Unable to complete flush {ENCODED => 7c7f87a6124bfe2f4049ee7803e0f98c, NAME => 'TestAcidGuarantees,,1731751108989.7c7f87a6124bfe2f4049ee7803e0f98c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T09:58:33,409 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37877 {}] master.HMaster(4177): Remote procedure failed, pid=39 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 7c7f87a6124bfe2f4049ee7803e0f98c, NAME => 'TestAcidGuarantees,,1731751108989.7c7f87a6124bfe2f4049ee7803e0f98c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 7c7f87a6124bfe2f4049ee7803e0f98c, NAME => 'TestAcidGuarantees,,1731751108989.7c7f87a6124bfe2f4049ee7803e0f98c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T09:58:33,413 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411169f1b7c09e8fd45fbae6889443a7ede76_7c7f87a6124bfe2f4049ee7803e0f98c is 50, key is test_row_0/A:col10/1731751112255/Put/seqid=0 2024-11-16T09:58:33,423 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c7f87a6124bfe2f4049ee7803e0f98c, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T09:58:33,423 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52392 deadline: 1731751173417, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c7f87a6124bfe2f4049ee7803e0f98c, server=037c58a19ddb,37267,1731751077844 2024-11-16T09:58:33,423 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c7f87a6124bfe2f4049ee7803e0f98c, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T09:58:33,424 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52420 deadline: 1731751173420, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c7f87a6124bfe2f4049ee7803e0f98c, server=037c58a19ddb,37267,1731751077844 2024-11-16T09:58:33,424 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731751108989.7c7f87a6124bfe2f4049ee7803e0f98c., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 , the old value is region=TestAcidGuarantees,,1731751108989.7c7f87a6124bfe2f4049ee7803e0f98c., hostname=037c58a19ddb,37267,1731751077844, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c7f87a6124bfe2f4049ee7803e0f98c, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:58:33,424 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731751108989.7c7f87a6124bfe2f4049ee7803e0f98c., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c7f87a6124bfe2f4049ee7803e0f98c, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:58:33,424 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731751108989.7c7f87a6124bfe2f4049ee7803e0f98c., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 because the exception is null or not the one we care about 2024-11-16T09:58:33,424 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731751108989.7c7f87a6124bfe2f4049ee7803e0f98c., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 , the old value is region=TestAcidGuarantees,,1731751108989.7c7f87a6124bfe2f4049ee7803e0f98c., hostname=037c58a19ddb,37267,1731751077844, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c7f87a6124bfe2f4049ee7803e0f98c, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:58:33,425 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731751108989.7c7f87a6124bfe2f4049ee7803e0f98c., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c7f87a6124bfe2f4049ee7803e0f98c, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:58:33,425 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731751108989.7c7f87a6124bfe2f4049ee7803e0f98c., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 because the exception is null or not the one we care about 2024-11-16T09:58:33,430 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c7f87a6124bfe2f4049ee7803e0f98c, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T09:58:33,430 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52428 deadline: 1731751173423, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c7f87a6124bfe2f4049ee7803e0f98c, server=037c58a19ddb,37267,1731751077844 2024-11-16T09:58:33,430 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c7f87a6124bfe2f4049ee7803e0f98c, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T09:58:33,430 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52396 deadline: 1731751173424, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c7f87a6124bfe2f4049ee7803e0f98c, server=037c58a19ddb,37267,1731751077844 2024-11-16T09:58:33,431 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731751108989.7c7f87a6124bfe2f4049ee7803e0f98c., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 , the old value is region=TestAcidGuarantees,,1731751108989.7c7f87a6124bfe2f4049ee7803e0f98c., hostname=037c58a19ddb,37267,1731751077844, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c7f87a6124bfe2f4049ee7803e0f98c, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:58:33,431 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731751108989.7c7f87a6124bfe2f4049ee7803e0f98c., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c7f87a6124bfe2f4049ee7803e0f98c, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:58:33,431 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731751108989.7c7f87a6124bfe2f4049ee7803e0f98c., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 because the exception is null or not the one we care about 2024-11-16T09:58:33,432 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c7f87a6124bfe2f4049ee7803e0f98c, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T09:58:33,432 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52400 deadline: 1731751173425, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c7f87a6124bfe2f4049ee7803e0f98c, server=037c58a19ddb,37267,1731751077844 2024-11-16T09:58:33,432 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731751108989.7c7f87a6124bfe2f4049ee7803e0f98c., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 , the old value is region=TestAcidGuarantees,,1731751108989.7c7f87a6124bfe2f4049ee7803e0f98c., hostname=037c58a19ddb,37267,1731751077844, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c7f87a6124bfe2f4049ee7803e0f98c, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:58:33,432 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731751108989.7c7f87a6124bfe2f4049ee7803e0f98c., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c7f87a6124bfe2f4049ee7803e0f98c, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:58:33,432 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731751108989.7c7f87a6124bfe2f4049ee7803e0f98c., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 because the exception is null or not the one we care about 2024-11-16T09:58:33,432 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731751108989.7c7f87a6124bfe2f4049ee7803e0f98c., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 , the old value is region=TestAcidGuarantees,,1731751108989.7c7f87a6124bfe2f4049ee7803e0f98c., hostname=037c58a19ddb,37267,1731751077844, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c7f87a6124bfe2f4049ee7803e0f98c, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:58:33,432 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731751108989.7c7f87a6124bfe2f4049ee7803e0f98c., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c7f87a6124bfe2f4049ee7803e0f98c, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:58:33,432 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731751108989.7c7f87a6124bfe2f4049ee7803e0f98c., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 because the exception is null or not the one we care about 2024-11-16T09:58:33,449 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37591 is added to blk_1073741973_1149 (size=14594) 2024-11-16T09:58:33,534 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c7f87a6124bfe2f4049ee7803e0f98c, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T09:58:33,534 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52392 deadline: 1731751173534, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c7f87a6124bfe2f4049ee7803e0f98c, server=037c58a19ddb,37267,1731751077844 2024-11-16T09:58:33,535 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731751108989.7c7f87a6124bfe2f4049ee7803e0f98c., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 , the old value is region=TestAcidGuarantees,,1731751108989.7c7f87a6124bfe2f4049ee7803e0f98c., hostname=037c58a19ddb,37267,1731751077844, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c7f87a6124bfe2f4049ee7803e0f98c, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:58:33,535 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731751108989.7c7f87a6124bfe2f4049ee7803e0f98c., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c7f87a6124bfe2f4049ee7803e0f98c, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:58:33,535 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731751108989.7c7f87a6124bfe2f4049ee7803e0f98c., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 because the exception is null or not the one we care about 2024-11-16T09:58:33,535 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c7f87a6124bfe2f4049ee7803e0f98c, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T09:58:33,536 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52420 deadline: 1731751173535, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c7f87a6124bfe2f4049ee7803e0f98c, server=037c58a19ddb,37267,1731751077844 2024-11-16T09:58:33,536 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731751108989.7c7f87a6124bfe2f4049ee7803e0f98c., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 , the old value is region=TestAcidGuarantees,,1731751108989.7c7f87a6124bfe2f4049ee7803e0f98c., hostname=037c58a19ddb,37267,1731751077844, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c7f87a6124bfe2f4049ee7803e0f98c, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:58:33,536 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731751108989.7c7f87a6124bfe2f4049ee7803e0f98c., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c7f87a6124bfe2f4049ee7803e0f98c, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:58:33,536 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731751108989.7c7f87a6124bfe2f4049ee7803e0f98c., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 because the exception is null or not the one we care about 2024-11-16T09:58:33,545 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c7f87a6124bfe2f4049ee7803e0f98c, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T09:58:33,545 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52396 deadline: 1731751173543, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c7f87a6124bfe2f4049ee7803e0f98c, server=037c58a19ddb,37267,1731751077844 2024-11-16T09:58:33,545 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731751108989.7c7f87a6124bfe2f4049ee7803e0f98c., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 , the old value is region=TestAcidGuarantees,,1731751108989.7c7f87a6124bfe2f4049ee7803e0f98c., hostname=037c58a19ddb,37267,1731751077844, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c7f87a6124bfe2f4049ee7803e0f98c, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:58:33,546 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731751108989.7c7f87a6124bfe2f4049ee7803e0f98c., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c7f87a6124bfe2f4049ee7803e0f98c, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:58:33,546 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731751108989.7c7f87a6124bfe2f4049ee7803e0f98c., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 because the exception is null or not the one we care about 2024-11-16T09:58:33,547 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c7f87a6124bfe2f4049ee7803e0f98c, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T09:58:33,547 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52428 deadline: 1731751173543, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c7f87a6124bfe2f4049ee7803e0f98c, server=037c58a19ddb,37267,1731751077844 2024-11-16T09:58:33,548 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c7f87a6124bfe2f4049ee7803e0f98c, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T09:58:33,548 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731751108989.7c7f87a6124bfe2f4049ee7803e0f98c., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 , the old value is region=TestAcidGuarantees,,1731751108989.7c7f87a6124bfe2f4049ee7803e0f98c., hostname=037c58a19ddb,37267,1731751077844, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c7f87a6124bfe2f4049ee7803e0f98c, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:58:33,548 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731751108989.7c7f87a6124bfe2f4049ee7803e0f98c., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c7f87a6124bfe2f4049ee7803e0f98c, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:58:33,548 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731751108989.7c7f87a6124bfe2f4049ee7803e0f98c., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 because the exception is null or not the one we care about 2024-11-16T09:58:33,548 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52400 deadline: 1731751173543, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c7f87a6124bfe2f4049ee7803e0f98c, server=037c58a19ddb,37267,1731751077844 2024-11-16T09:58:33,549 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731751108989.7c7f87a6124bfe2f4049ee7803e0f98c., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 , the old value is region=TestAcidGuarantees,,1731751108989.7c7f87a6124bfe2f4049ee7803e0f98c., hostname=037c58a19ddb,37267,1731751077844, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c7f87a6124bfe2f4049ee7803e0f98c, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:58:33,549 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731751108989.7c7f87a6124bfe2f4049ee7803e0f98c., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c7f87a6124bfe2f4049ee7803e0f98c, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:58:33,549 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731751108989.7c7f87a6124bfe2f4049ee7803e0f98c., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 because the exception is null or not the one we care about 2024-11-16T09:58:33,562 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=37267 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=39 2024-11-16T09:58:33,562 DEBUG [RS_FLUSH_OPERATIONS-regionserver/037c58a19ddb:0-0 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731751108989.7c7f87a6124bfe2f4049ee7803e0f98c. 2024-11-16T09:58:33,562 DEBUG [RS_FLUSH_OPERATIONS-regionserver/037c58a19ddb:0-0 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731751108989.7c7f87a6124bfe2f4049ee7803e0f98c. as already flushing 2024-11-16T09:58:33,562 DEBUG [RS_FLUSH_OPERATIONS-regionserver/037c58a19ddb:0-0 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731751108989.7c7f87a6124bfe2f4049ee7803e0f98c. 2024-11-16T09:58:33,562 ERROR [RS_FLUSH_OPERATIONS-regionserver/037c58a19ddb:0-0 {event_type=RS_FLUSH_REGIONS, pid=39}] handler.RSProcedureHandler(58): pid=39 java.io.IOException: Unable to complete flush {ENCODED => 7c7f87a6124bfe2f4049ee7803e0f98c, NAME => 'TestAcidGuarantees,,1731751108989.7c7f87a6124bfe2f4049ee7803e0f98c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T09:58:33,563 DEBUG [RS_FLUSH_OPERATIONS-regionserver/037c58a19ddb:0-0 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=39 java.io.IOException: Unable to complete flush {ENCODED => 7c7f87a6124bfe2f4049ee7803e0f98c, NAME => 'TestAcidGuarantees,,1731751108989.7c7f87a6124bfe2f4049ee7803e0f98c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T09:58:33,563 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37877 {}] master.HMaster(4177): Remote procedure failed, pid=39 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 7c7f87a6124bfe2f4049ee7803e0f98c, NAME => 'TestAcidGuarantees,,1731751108989.7c7f87a6124bfe2f4049ee7803e0f98c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 7c7f87a6124bfe2f4049ee7803e0f98c, NAME => 'TestAcidGuarantees,,1731751108989.7c7f87a6124bfe2f4049ee7803e0f98c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T09:58:33,718 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=37267 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=39 2024-11-16T09:58:33,719 DEBUG [RS_FLUSH_OPERATIONS-regionserver/037c58a19ddb:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731751108989.7c7f87a6124bfe2f4049ee7803e0f98c. 2024-11-16T09:58:33,719 DEBUG [RS_FLUSH_OPERATIONS-regionserver/037c58a19ddb:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731751108989.7c7f87a6124bfe2f4049ee7803e0f98c. as already flushing 2024-11-16T09:58:33,719 DEBUG [RS_FLUSH_OPERATIONS-regionserver/037c58a19ddb:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731751108989.7c7f87a6124bfe2f4049ee7803e0f98c. 2024-11-16T09:58:33,719 ERROR [RS_FLUSH_OPERATIONS-regionserver/037c58a19ddb:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] handler.RSProcedureHandler(58): pid=39 java.io.IOException: Unable to complete flush {ENCODED => 7c7f87a6124bfe2f4049ee7803e0f98c, NAME => 'TestAcidGuarantees,,1731751108989.7c7f87a6124bfe2f4049ee7803e0f98c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T09:58:33,720 DEBUG [RS_FLUSH_OPERATIONS-regionserver/037c58a19ddb:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=39 java.io.IOException: Unable to complete flush {ENCODED => 7c7f87a6124bfe2f4049ee7803e0f98c, NAME => 'TestAcidGuarantees,,1731751108989.7c7f87a6124bfe2f4049ee7803e0f98c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T09:58:33,721 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37877 {}] master.HMaster(4177): Remote procedure failed, pid=39 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 7c7f87a6124bfe2f4049ee7803e0f98c, NAME => 'TestAcidGuarantees,,1731751108989.7c7f87a6124bfe2f4049ee7803e0f98c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 7c7f87a6124bfe2f4049ee7803e0f98c, NAME => 'TestAcidGuarantees,,1731751108989.7c7f87a6124bfe2f4049ee7803e0f98c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T09:58:33,743 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c7f87a6124bfe2f4049ee7803e0f98c, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T09:58:33,744 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52392 deadline: 1731751173743, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c7f87a6124bfe2f4049ee7803e0f98c, server=037c58a19ddb,37267,1731751077844 2024-11-16T09:58:33,744 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c7f87a6124bfe2f4049ee7803e0f98c, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T09:58:33,744 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731751108989.7c7f87a6124bfe2f4049ee7803e0f98c., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 , the old value is region=TestAcidGuarantees,,1731751108989.7c7f87a6124bfe2f4049ee7803e0f98c., hostname=037c58a19ddb,37267,1731751077844, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c7f87a6124bfe2f4049ee7803e0f98c, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:58:33,744 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52420 deadline: 1731751173743, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c7f87a6124bfe2f4049ee7803e0f98c, server=037c58a19ddb,37267,1731751077844 2024-11-16T09:58:33,744 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731751108989.7c7f87a6124bfe2f4049ee7803e0f98c., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c7f87a6124bfe2f4049ee7803e0f98c, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:58:33,744 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731751108989.7c7f87a6124bfe2f4049ee7803e0f98c., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 because the exception is null or not the one we care about 2024-11-16T09:58:33,745 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731751108989.7c7f87a6124bfe2f4049ee7803e0f98c., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 , the old value is region=TestAcidGuarantees,,1731751108989.7c7f87a6124bfe2f4049ee7803e0f98c., hostname=037c58a19ddb,37267,1731751077844, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c7f87a6124bfe2f4049ee7803e0f98c, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:58:33,745 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731751108989.7c7f87a6124bfe2f4049ee7803e0f98c., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c7f87a6124bfe2f4049ee7803e0f98c, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:58:33,745 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731751108989.7c7f87a6124bfe2f4049ee7803e0f98c., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 because the exception is null or not the one we care about 2024-11-16T09:58:33,756 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c7f87a6124bfe2f4049ee7803e0f98c, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T09:58:33,756 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52396 deadline: 1731751173753, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c7f87a6124bfe2f4049ee7803e0f98c, server=037c58a19ddb,37267,1731751077844 2024-11-16T09:58:33,756 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c7f87a6124bfe2f4049ee7803e0f98c, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T09:58:33,757 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52400 deadline: 1731751173753, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c7f87a6124bfe2f4049ee7803e0f98c, server=037c58a19ddb,37267,1731751077844 2024-11-16T09:58:33,757 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c7f87a6124bfe2f4049ee7803e0f98c, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T09:58:33,757 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52428 deadline: 1731751173753, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c7f87a6124bfe2f4049ee7803e0f98c, server=037c58a19ddb,37267,1731751077844 2024-11-16T09:58:33,757 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731751108989.7c7f87a6124bfe2f4049ee7803e0f98c., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 , the old value is region=TestAcidGuarantees,,1731751108989.7c7f87a6124bfe2f4049ee7803e0f98c., hostname=037c58a19ddb,37267,1731751077844, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c7f87a6124bfe2f4049ee7803e0f98c, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:58:33,757 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731751108989.7c7f87a6124bfe2f4049ee7803e0f98c., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 , the old value is region=TestAcidGuarantees,,1731751108989.7c7f87a6124bfe2f4049ee7803e0f98c., hostname=037c58a19ddb,37267,1731751077844, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c7f87a6124bfe2f4049ee7803e0f98c, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:58:33,757 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731751108989.7c7f87a6124bfe2f4049ee7803e0f98c., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c7f87a6124bfe2f4049ee7803e0f98c, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:58:33,758 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731751108989.7c7f87a6124bfe2f4049ee7803e0f98c., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c7f87a6124bfe2f4049ee7803e0f98c, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:58:33,758 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731751108989.7c7f87a6124bfe2f4049ee7803e0f98c., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 because the exception is null or not the one we care about 2024-11-16T09:58:33,758 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731751108989.7c7f87a6124bfe2f4049ee7803e0f98c., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 because the exception is null or not the one we care about 2024-11-16T09:58:33,758 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731751108989.7c7f87a6124bfe2f4049ee7803e0f98c., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 , the old value is region=TestAcidGuarantees,,1731751108989.7c7f87a6124bfe2f4049ee7803e0f98c., hostname=037c58a19ddb,37267,1731751077844, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c7f87a6124bfe2f4049ee7803e0f98c, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:58:33,758 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731751108989.7c7f87a6124bfe2f4049ee7803e0f98c., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c7f87a6124bfe2f4049ee7803e0f98c, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:58:33,758 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731751108989.7c7f87a6124bfe2f4049ee7803e0f98c., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 because the exception is null or not the one we care about 2024-11-16T09:58:33,847 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-11-16T09:58:33,851 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T09:58:33,858 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411169f1b7c09e8fd45fbae6889443a7ede76_7c7f87a6124bfe2f4049ee7803e0f98c to hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411169f1b7c09e8fd45fbae6889443a7ede76_7c7f87a6124bfe2f4049ee7803e0f98c 2024-11-16T09:58:33,859 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/7c7f87a6124bfe2f4049ee7803e0f98c/.tmp/A/ee8a20ddfc1549a1b4d7045964ed8bb8, store: [table=TestAcidGuarantees family=A region=7c7f87a6124bfe2f4049ee7803e0f98c] 2024-11-16T09:58:33,860 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/7c7f87a6124bfe2f4049ee7803e0f98c/.tmp/A/ee8a20ddfc1549a1b4d7045964ed8bb8 is 175, key is test_row_0/A:col10/1731751112255/Put/seqid=0 2024-11-16T09:58:33,874 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=37267 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=39 2024-11-16T09:58:33,874 DEBUG [RS_FLUSH_OPERATIONS-regionserver/037c58a19ddb:0-2 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731751108989.7c7f87a6124bfe2f4049ee7803e0f98c. 2024-11-16T09:58:33,874 DEBUG [RS_FLUSH_OPERATIONS-regionserver/037c58a19ddb:0-2 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731751108989.7c7f87a6124bfe2f4049ee7803e0f98c. as already flushing 2024-11-16T09:58:33,874 DEBUG [RS_FLUSH_OPERATIONS-regionserver/037c58a19ddb:0-2 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731751108989.7c7f87a6124bfe2f4049ee7803e0f98c. 2024-11-16T09:58:33,874 ERROR [RS_FLUSH_OPERATIONS-regionserver/037c58a19ddb:0-2 {event_type=RS_FLUSH_REGIONS, pid=39}] handler.RSProcedureHandler(58): pid=39 java.io.IOException: Unable to complete flush {ENCODED => 7c7f87a6124bfe2f4049ee7803e0f98c, NAME => 'TestAcidGuarantees,,1731751108989.7c7f87a6124bfe2f4049ee7803e0f98c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T09:58:33,875 DEBUG [RS_FLUSH_OPERATIONS-regionserver/037c58a19ddb:0-2 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=39 java.io.IOException: Unable to complete flush {ENCODED => 7c7f87a6124bfe2f4049ee7803e0f98c, NAME => 'TestAcidGuarantees,,1731751108989.7c7f87a6124bfe2f4049ee7803e0f98c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T09:58:33,881 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37877 {}] master.HMaster(4177): Remote procedure failed, pid=39 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 7c7f87a6124bfe2f4049ee7803e0f98c, NAME => 'TestAcidGuarantees,,1731751108989.7c7f87a6124bfe2f4049ee7803e0f98c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 7c7f87a6124bfe2f4049ee7803e0f98c, NAME => 'TestAcidGuarantees,,1731751108989.7c7f87a6124bfe2f4049ee7803e0f98c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T09:58:33,919 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37591 is added to blk_1073741974_1150 (size=39549) 2024-11-16T09:58:33,920 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=41, memsize=47.0 K, hasBloomFilter=true, into tmp file hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/7c7f87a6124bfe2f4049ee7803e0f98c/.tmp/A/ee8a20ddfc1549a1b4d7045964ed8bb8 2024-11-16T09:58:33,949 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/7c7f87a6124bfe2f4049ee7803e0f98c/.tmp/B/5e27822b25b0488bb4324b737dfd6709 is 50, key is test_row_0/B:col10/1731751112255/Put/seqid=0 2024-11-16T09:58:33,974 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37591 is added to blk_1073741975_1151 (size=12001) 2024-11-16T09:58:33,976 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=41 (bloomFilter=true), to=hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/7c7f87a6124bfe2f4049ee7803e0f98c/.tmp/B/5e27822b25b0488bb4324b737dfd6709 2024-11-16T09:58:33,990 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/7c7f87a6124bfe2f4049ee7803e0f98c/.tmp/C/95998a9414674875803abd21d398ade9 is 50, key is test_row_0/C:col10/1731751112255/Put/seqid=0 2024-11-16T09:58:34,004 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37591 is added to blk_1073741976_1152 (size=12001) 2024-11-16T09:58:34,034 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=37267 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=39 2024-11-16T09:58:34,035 DEBUG [RS_FLUSH_OPERATIONS-regionserver/037c58a19ddb:0-0 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731751108989.7c7f87a6124bfe2f4049ee7803e0f98c. 2024-11-16T09:58:34,035 DEBUG [RS_FLUSH_OPERATIONS-regionserver/037c58a19ddb:0-0 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731751108989.7c7f87a6124bfe2f4049ee7803e0f98c. as already flushing 2024-11-16T09:58:34,035 DEBUG [RS_FLUSH_OPERATIONS-regionserver/037c58a19ddb:0-0 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731751108989.7c7f87a6124bfe2f4049ee7803e0f98c. 2024-11-16T09:58:34,036 ERROR [RS_FLUSH_OPERATIONS-regionserver/037c58a19ddb:0-0 {event_type=RS_FLUSH_REGIONS, pid=39}] handler.RSProcedureHandler(58): pid=39 java.io.IOException: Unable to complete flush {ENCODED => 7c7f87a6124bfe2f4049ee7803e0f98c, NAME => 'TestAcidGuarantees,,1731751108989.7c7f87a6124bfe2f4049ee7803e0f98c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T09:58:34,036 DEBUG [RS_FLUSH_OPERATIONS-regionserver/037c58a19ddb:0-0 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=39 java.io.IOException: Unable to complete flush {ENCODED => 7c7f87a6124bfe2f4049ee7803e0f98c, NAME => 'TestAcidGuarantees,,1731751108989.7c7f87a6124bfe2f4049ee7803e0f98c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T09:58:34,038 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37877 {}] master.HMaster(4177): Remote procedure failed, pid=39 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 7c7f87a6124bfe2f4049ee7803e0f98c, NAME => 'TestAcidGuarantees,,1731751108989.7c7f87a6124bfe2f4049ee7803e0f98c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 7c7f87a6124bfe2f4049ee7803e0f98c, NAME => 'TestAcidGuarantees,,1731751108989.7c7f87a6124bfe2f4049ee7803e0f98c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T09:58:34,055 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c7f87a6124bfe2f4049ee7803e0f98c, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T09:58:34,055 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52392 deadline: 1731751174053, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c7f87a6124bfe2f4049ee7803e0f98c, server=037c58a19ddb,37267,1731751077844 2024-11-16T09:58:34,056 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c7f87a6124bfe2f4049ee7803e0f98c, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T09:58:34,056 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52420 deadline: 1731751174053, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c7f87a6124bfe2f4049ee7803e0f98c, server=037c58a19ddb,37267,1731751077844 2024-11-16T09:58:34,056 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731751108989.7c7f87a6124bfe2f4049ee7803e0f98c., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 , the old value is region=TestAcidGuarantees,,1731751108989.7c7f87a6124bfe2f4049ee7803e0f98c., hostname=037c58a19ddb,37267,1731751077844, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c7f87a6124bfe2f4049ee7803e0f98c, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:58:34,056 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731751108989.7c7f87a6124bfe2f4049ee7803e0f98c., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c7f87a6124bfe2f4049ee7803e0f98c, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:58:34,056 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731751108989.7c7f87a6124bfe2f4049ee7803e0f98c., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 because the exception is null or not the one we care about 2024-11-16T09:58:34,056 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731751108989.7c7f87a6124bfe2f4049ee7803e0f98c., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 , the old value is region=TestAcidGuarantees,,1731751108989.7c7f87a6124bfe2f4049ee7803e0f98c., hostname=037c58a19ddb,37267,1731751077844, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c7f87a6124bfe2f4049ee7803e0f98c, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:58:34,056 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731751108989.7c7f87a6124bfe2f4049ee7803e0f98c., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c7f87a6124bfe2f4049ee7803e0f98c, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:58:34,057 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731751108989.7c7f87a6124bfe2f4049ee7803e0f98c., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 because the exception is null or not the one we care about 2024-11-16T09:58:34,063 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c7f87a6124bfe2f4049ee7803e0f98c, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T09:58:34,064 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52396 deadline: 1731751174063, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c7f87a6124bfe2f4049ee7803e0f98c, server=037c58a19ddb,37267,1731751077844 2024-11-16T09:58:34,064 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c7f87a6124bfe2f4049ee7803e0f98c, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T09:58:34,064 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c7f87a6124bfe2f4049ee7803e0f98c, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T09:58:34,064 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52428 deadline: 1731751174063, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c7f87a6124bfe2f4049ee7803e0f98c, server=037c58a19ddb,37267,1731751077844 2024-11-16T09:58:34,064 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52400 deadline: 1731751174063, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c7f87a6124bfe2f4049ee7803e0f98c, server=037c58a19ddb,37267,1731751077844 2024-11-16T09:58:34,065 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731751108989.7c7f87a6124bfe2f4049ee7803e0f98c., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 , the old value is region=TestAcidGuarantees,,1731751108989.7c7f87a6124bfe2f4049ee7803e0f98c., hostname=037c58a19ddb,37267,1731751077844, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c7f87a6124bfe2f4049ee7803e0f98c, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:58:34,065 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731751108989.7c7f87a6124bfe2f4049ee7803e0f98c., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c7f87a6124bfe2f4049ee7803e0f98c, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:58:34,065 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731751108989.7c7f87a6124bfe2f4049ee7803e0f98c., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 because the exception is null or not the one we care about 2024-11-16T09:58:34,066 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731751108989.7c7f87a6124bfe2f4049ee7803e0f98c., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 , the old value is region=TestAcidGuarantees,,1731751108989.7c7f87a6124bfe2f4049ee7803e0f98c., hostname=037c58a19ddb,37267,1731751077844, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c7f87a6124bfe2f4049ee7803e0f98c, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:58:34,066 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731751108989.7c7f87a6124bfe2f4049ee7803e0f98c., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c7f87a6124bfe2f4049ee7803e0f98c, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:58:34,066 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731751108989.7c7f87a6124bfe2f4049ee7803e0f98c., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 because the exception is null or not the one we care about 2024-11-16T09:58:34,066 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731751108989.7c7f87a6124bfe2f4049ee7803e0f98c., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 , the old value is region=TestAcidGuarantees,,1731751108989.7c7f87a6124bfe2f4049ee7803e0f98c., hostname=037c58a19ddb,37267,1731751077844, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c7f87a6124bfe2f4049ee7803e0f98c, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:58:34,066 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731751108989.7c7f87a6124bfe2f4049ee7803e0f98c., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c7f87a6124bfe2f4049ee7803e0f98c, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:58:34,066 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731751108989.7c7f87a6124bfe2f4049ee7803e0f98c., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 because the exception is null or not the one we care about 2024-11-16T09:58:34,191 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=37267 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=39 2024-11-16T09:58:34,191 DEBUG [RS_FLUSH_OPERATIONS-regionserver/037c58a19ddb:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731751108989.7c7f87a6124bfe2f4049ee7803e0f98c. 2024-11-16T09:58:34,192 DEBUG [RS_FLUSH_OPERATIONS-regionserver/037c58a19ddb:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731751108989.7c7f87a6124bfe2f4049ee7803e0f98c. as already flushing 2024-11-16T09:58:34,192 DEBUG [RS_FLUSH_OPERATIONS-regionserver/037c58a19ddb:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731751108989.7c7f87a6124bfe2f4049ee7803e0f98c. 2024-11-16T09:58:34,192 ERROR [RS_FLUSH_OPERATIONS-regionserver/037c58a19ddb:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] handler.RSProcedureHandler(58): pid=39 java.io.IOException: Unable to complete flush {ENCODED => 7c7f87a6124bfe2f4049ee7803e0f98c, NAME => 'TestAcidGuarantees,,1731751108989.7c7f87a6124bfe2f4049ee7803e0f98c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T09:58:34,192 DEBUG [RS_FLUSH_OPERATIONS-regionserver/037c58a19ddb:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=39 java.io.IOException: Unable to complete flush {ENCODED => 7c7f87a6124bfe2f4049ee7803e0f98c, NAME => 'TestAcidGuarantees,,1731751108989.7c7f87a6124bfe2f4049ee7803e0f98c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T09:58:34,193 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37877 {}] master.HMaster(4177): Remote procedure failed, pid=39 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 7c7f87a6124bfe2f4049ee7803e0f98c, NAME => 'TestAcidGuarantees,,1731751108989.7c7f87a6124bfe2f4049ee7803e0f98c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 7c7f87a6124bfe2f4049ee7803e0f98c, NAME => 'TestAcidGuarantees,,1731751108989.7c7f87a6124bfe2f4049ee7803e0f98c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T09:58:34,313 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37877 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=38 2024-11-16T09:58:34,345 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=37267 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=39 2024-11-16T09:58:34,346 DEBUG [RS_FLUSH_OPERATIONS-regionserver/037c58a19ddb:0-2 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731751108989.7c7f87a6124bfe2f4049ee7803e0f98c. 2024-11-16T09:58:34,346 DEBUG [RS_FLUSH_OPERATIONS-regionserver/037c58a19ddb:0-2 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731751108989.7c7f87a6124bfe2f4049ee7803e0f98c. as already flushing 2024-11-16T09:58:34,346 DEBUG [RS_FLUSH_OPERATIONS-regionserver/037c58a19ddb:0-2 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731751108989.7c7f87a6124bfe2f4049ee7803e0f98c. 2024-11-16T09:58:34,346 ERROR [RS_FLUSH_OPERATIONS-regionserver/037c58a19ddb:0-2 {event_type=RS_FLUSH_REGIONS, pid=39}] handler.RSProcedureHandler(58): pid=39 java.io.IOException: Unable to complete flush {ENCODED => 7c7f87a6124bfe2f4049ee7803e0f98c, NAME => 'TestAcidGuarantees,,1731751108989.7c7f87a6124bfe2f4049ee7803e0f98c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T09:58:34,346 DEBUG [RS_FLUSH_OPERATIONS-regionserver/037c58a19ddb:0-2 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=39 java.io.IOException: Unable to complete flush {ENCODED => 7c7f87a6124bfe2f4049ee7803e0f98c, NAME => 'TestAcidGuarantees,,1731751108989.7c7f87a6124bfe2f4049ee7803e0f98c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T09:58:34,347 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37877 {}] master.HMaster(4177): Remote procedure failed, pid=39 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 7c7f87a6124bfe2f4049ee7803e0f98c, NAME => 'TestAcidGuarantees,,1731751108989.7c7f87a6124bfe2f4049ee7803e0f98c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 7c7f87a6124bfe2f4049ee7803e0f98c, NAME => 'TestAcidGuarantees,,1731751108989.7c7f87a6124bfe2f4049ee7803e0f98c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T09:58:34,409 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=41 (bloomFilter=true), to=hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/7c7f87a6124bfe2f4049ee7803e0f98c/.tmp/C/95998a9414674875803abd21d398ade9 2024-11-16T09:58:34,416 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/7c7f87a6124bfe2f4049ee7803e0f98c/.tmp/A/ee8a20ddfc1549a1b4d7045964ed8bb8 as hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/7c7f87a6124bfe2f4049ee7803e0f98c/A/ee8a20ddfc1549a1b4d7045964ed8bb8 2024-11-16T09:58:34,421 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/7c7f87a6124bfe2f4049ee7803e0f98c/A/ee8a20ddfc1549a1b4d7045964ed8bb8, entries=200, sequenceid=41, filesize=38.6 K 2024-11-16T09:58:34,423 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/7c7f87a6124bfe2f4049ee7803e0f98c/.tmp/B/5e27822b25b0488bb4324b737dfd6709 as hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/7c7f87a6124bfe2f4049ee7803e0f98c/B/5e27822b25b0488bb4324b737dfd6709 2024-11-16T09:58:34,431 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/7c7f87a6124bfe2f4049ee7803e0f98c/B/5e27822b25b0488bb4324b737dfd6709, entries=150, sequenceid=41, filesize=11.7 K 2024-11-16T09:58:34,434 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/7c7f87a6124bfe2f4049ee7803e0f98c/.tmp/C/95998a9414674875803abd21d398ade9 as hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/7c7f87a6124bfe2f4049ee7803e0f98c/C/95998a9414674875803abd21d398ade9 2024-11-16T09:58:34,440 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/7c7f87a6124bfe2f4049ee7803e0f98c/C/95998a9414674875803abd21d398ade9, entries=150, sequenceid=41, filesize=11.7 K 2024-11-16T09:58:34,442 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~140.89 KB/144270, heapSize ~369.84 KB/378720, currentSize=60.38 KB/61830 for 7c7f87a6124bfe2f4049ee7803e0f98c in 1045ms, sequenceid=41, compaction requested=false 2024-11-16T09:58:34,442 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 7c7f87a6124bfe2f4049ee7803e0f98c: 2024-11-16T09:58:34,499 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=37267 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=39 2024-11-16T09:58:34,499 DEBUG [RS_FLUSH_OPERATIONS-regionserver/037c58a19ddb:0-0 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731751108989.7c7f87a6124bfe2f4049ee7803e0f98c. 2024-11-16T09:58:34,500 INFO [RS_FLUSH_OPERATIONS-regionserver/037c58a19ddb:0-0 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.HRegion(2902): Flushing 7c7f87a6124bfe2f4049ee7803e0f98c 3/3 column families, dataSize=60.38 KB heapSize=158.95 KB 2024-11-16T09:58:34,508 DEBUG [RS_FLUSH_OPERATIONS-regionserver/037c58a19ddb:0-0 {event_type=RS_FLUSH_REGIONS, pid=39}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111693d75b69d9444da09a3e69d0d73ffec7_7c7f87a6124bfe2f4049ee7803e0f98c is 50, key is test_row_0/A:col10/1731751113405/Put/seqid=0 2024-11-16T09:58:34,519 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37591 is added to blk_1073741977_1153 (size=12154) 2024-11-16T09:58:34,566 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] regionserver.HRegion(8855): Flush requested on 7c7f87a6124bfe2f4049ee7803e0f98c 2024-11-16T09:58:34,566 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731751108989.7c7f87a6124bfe2f4049ee7803e0f98c. as already flushing 2024-11-16T09:58:34,605 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c7f87a6124bfe2f4049ee7803e0f98c, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T09:58:34,606 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52400 deadline: 1731751174600, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c7f87a6124bfe2f4049ee7803e0f98c, server=037c58a19ddb,37267,1731751077844 2024-11-16T09:58:34,606 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731751108989.7c7f87a6124bfe2f4049ee7803e0f98c., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 , the old value is region=TestAcidGuarantees,,1731751108989.7c7f87a6124bfe2f4049ee7803e0f98c., hostname=037c58a19ddb,37267,1731751077844, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c7f87a6124bfe2f4049ee7803e0f98c, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:58:34,606 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731751108989.7c7f87a6124bfe2f4049ee7803e0f98c., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c7f87a6124bfe2f4049ee7803e0f98c, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:58:34,606 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731751108989.7c7f87a6124bfe2f4049ee7803e0f98c., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 because the exception is null or not the one we care about 2024-11-16T09:58:34,609 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c7f87a6124bfe2f4049ee7803e0f98c, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T09:58:34,609 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52420 deadline: 1731751174602, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c7f87a6124bfe2f4049ee7803e0f98c, server=037c58a19ddb,37267,1731751077844 2024-11-16T09:58:34,609 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c7f87a6124bfe2f4049ee7803e0f98c, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T09:58:34,610 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52392 deadline: 1731751174603, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c7f87a6124bfe2f4049ee7803e0f98c, server=037c58a19ddb,37267,1731751077844 2024-11-16T09:58:34,610 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731751108989.7c7f87a6124bfe2f4049ee7803e0f98c., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 , the old value is region=TestAcidGuarantees,,1731751108989.7c7f87a6124bfe2f4049ee7803e0f98c., hostname=037c58a19ddb,37267,1731751077844, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c7f87a6124bfe2f4049ee7803e0f98c, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:58:34,610 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731751108989.7c7f87a6124bfe2f4049ee7803e0f98c., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c7f87a6124bfe2f4049ee7803e0f98c, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:58:34,610 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c7f87a6124bfe2f4049ee7803e0f98c, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T09:58:34,610 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731751108989.7c7f87a6124bfe2f4049ee7803e0f98c., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 because the exception is null or not the one we care about 2024-11-16T09:58:34,610 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52396 deadline: 1731751174606, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c7f87a6124bfe2f4049ee7803e0f98c, server=037c58a19ddb,37267,1731751077844 2024-11-16T09:58:34,610 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c7f87a6124bfe2f4049ee7803e0f98c, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T09:58:34,610 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731751108989.7c7f87a6124bfe2f4049ee7803e0f98c., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 , the old value is region=TestAcidGuarantees,,1731751108989.7c7f87a6124bfe2f4049ee7803e0f98c., hostname=037c58a19ddb,37267,1731751077844, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c7f87a6124bfe2f4049ee7803e0f98c, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:58:34,610 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52428 deadline: 1731751174607, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c7f87a6124bfe2f4049ee7803e0f98c, server=037c58a19ddb,37267,1731751077844 2024-11-16T09:58:34,610 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731751108989.7c7f87a6124bfe2f4049ee7803e0f98c., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c7f87a6124bfe2f4049ee7803e0f98c, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:58:34,610 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731751108989.7c7f87a6124bfe2f4049ee7803e0f98c., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 because the exception is null or not the one we care about 2024-11-16T09:58:34,611 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731751108989.7c7f87a6124bfe2f4049ee7803e0f98c., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 , the old value is region=TestAcidGuarantees,,1731751108989.7c7f87a6124bfe2f4049ee7803e0f98c., hostname=037c58a19ddb,37267,1731751077844, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c7f87a6124bfe2f4049ee7803e0f98c, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:58:34,611 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731751108989.7c7f87a6124bfe2f4049ee7803e0f98c., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c7f87a6124bfe2f4049ee7803e0f98c, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:58:34,611 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731751108989.7c7f87a6124bfe2f4049ee7803e0f98c., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 because the exception is null or not the one we care about 2024-11-16T09:58:34,611 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731751108989.7c7f87a6124bfe2f4049ee7803e0f98c., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 , the old value is region=TestAcidGuarantees,,1731751108989.7c7f87a6124bfe2f4049ee7803e0f98c., hostname=037c58a19ddb,37267,1731751077844, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c7f87a6124bfe2f4049ee7803e0f98c, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:58:34,611 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731751108989.7c7f87a6124bfe2f4049ee7803e0f98c., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c7f87a6124bfe2f4049ee7803e0f98c, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:58:34,611 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731751108989.7c7f87a6124bfe2f4049ee7803e0f98c., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 because the exception is null or not the one we care about 2024-11-16T09:58:34,717 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c7f87a6124bfe2f4049ee7803e0f98c, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T09:58:34,717 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52400 deadline: 1731751174713, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c7f87a6124bfe2f4049ee7803e0f98c, server=037c58a19ddb,37267,1731751077844 2024-11-16T09:58:34,718 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731751108989.7c7f87a6124bfe2f4049ee7803e0f98c., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 , the old value is region=TestAcidGuarantees,,1731751108989.7c7f87a6124bfe2f4049ee7803e0f98c., hostname=037c58a19ddb,37267,1731751077844, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c7f87a6124bfe2f4049ee7803e0f98c, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:58:34,718 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731751108989.7c7f87a6124bfe2f4049ee7803e0f98c., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c7f87a6124bfe2f4049ee7803e0f98c, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:58:34,718 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731751108989.7c7f87a6124bfe2f4049ee7803e0f98c., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 because the exception is null or not the one we care about 2024-11-16T09:58:34,718 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c7f87a6124bfe2f4049ee7803e0f98c, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T09:58:34,718 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52420 deadline: 1731751174713, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c7f87a6124bfe2f4049ee7803e0f98c, server=037c58a19ddb,37267,1731751077844 2024-11-16T09:58:34,718 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c7f87a6124bfe2f4049ee7803e0f98c, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T09:58:34,718 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52428 deadline: 1731751174714, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c7f87a6124bfe2f4049ee7803e0f98c, server=037c58a19ddb,37267,1731751077844 2024-11-16T09:58:34,718 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731751108989.7c7f87a6124bfe2f4049ee7803e0f98c., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 , the old value is region=TestAcidGuarantees,,1731751108989.7c7f87a6124bfe2f4049ee7803e0f98c., hostname=037c58a19ddb,37267,1731751077844, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c7f87a6124bfe2f4049ee7803e0f98c, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:58:34,718 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731751108989.7c7f87a6124bfe2f4049ee7803e0f98c., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c7f87a6124bfe2f4049ee7803e0f98c, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:58:34,718 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731751108989.7c7f87a6124bfe2f4049ee7803e0f98c., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 because the exception is null or not the one we care about 2024-11-16T09:58:34,719 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731751108989.7c7f87a6124bfe2f4049ee7803e0f98c., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 , the old value is region=TestAcidGuarantees,,1731751108989.7c7f87a6124bfe2f4049ee7803e0f98c., hostname=037c58a19ddb,37267,1731751077844, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c7f87a6124bfe2f4049ee7803e0f98c, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:58:34,719 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731751108989.7c7f87a6124bfe2f4049ee7803e0f98c., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c7f87a6124bfe2f4049ee7803e0f98c, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:58:34,719 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731751108989.7c7f87a6124bfe2f4049ee7803e0f98c., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 because the exception is null or not the one we care about 2024-11-16T09:58:34,719 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c7f87a6124bfe2f4049ee7803e0f98c, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T09:58:34,719 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52396 deadline: 1731751174714, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c7f87a6124bfe2f4049ee7803e0f98c, server=037c58a19ddb,37267,1731751077844 2024-11-16T09:58:34,719 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731751108989.7c7f87a6124bfe2f4049ee7803e0f98c., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 , the old value is region=TestAcidGuarantees,,1731751108989.7c7f87a6124bfe2f4049ee7803e0f98c., hostname=037c58a19ddb,37267,1731751077844, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c7f87a6124bfe2f4049ee7803e0f98c, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:58:34,719 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731751108989.7c7f87a6124bfe2f4049ee7803e0f98c., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c7f87a6124bfe2f4049ee7803e0f98c, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:58:34,719 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731751108989.7c7f87a6124bfe2f4049ee7803e0f98c., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 because the exception is null or not the one we care about 2024-11-16T09:58:34,725 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c7f87a6124bfe2f4049ee7803e0f98c, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T09:58:34,725 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52392 deadline: 1731751174723, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c7f87a6124bfe2f4049ee7803e0f98c, server=037c58a19ddb,37267,1731751077844 2024-11-16T09:58:34,726 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731751108989.7c7f87a6124bfe2f4049ee7803e0f98c., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 , the old value is region=TestAcidGuarantees,,1731751108989.7c7f87a6124bfe2f4049ee7803e0f98c., hostname=037c58a19ddb,37267,1731751077844, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c7f87a6124bfe2f4049ee7803e0f98c, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:58:34,726 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731751108989.7c7f87a6124bfe2f4049ee7803e0f98c., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c7f87a6124bfe2f4049ee7803e0f98c, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:58:34,726 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731751108989.7c7f87a6124bfe2f4049ee7803e0f98c., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 because the exception is null or not the one we care about 2024-11-16T09:58:34,920 DEBUG [RS_FLUSH_OPERATIONS-regionserver/037c58a19ddb:0-0 {event_type=RS_FLUSH_REGIONS, pid=39}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T09:58:34,926 INFO [RS_FLUSH_OPERATIONS-regionserver/037c58a19ddb:0-0 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111693d75b69d9444da09a3e69d0d73ffec7_7c7f87a6124bfe2f4049ee7803e0f98c to hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111693d75b69d9444da09a3e69d0d73ffec7_7c7f87a6124bfe2f4049ee7803e0f98c 2024-11-16T09:58:34,928 DEBUG [RS_FLUSH_OPERATIONS-regionserver/037c58a19ddb:0-0 {event_type=RS_FLUSH_REGIONS, pid=39}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/7c7f87a6124bfe2f4049ee7803e0f98c/.tmp/A/a9b0a7760e764325a134f8bd6fd9c882, store: [table=TestAcidGuarantees family=A region=7c7f87a6124bfe2f4049ee7803e0f98c] 2024-11-16T09:58:34,929 DEBUG [RS_FLUSH_OPERATIONS-regionserver/037c58a19ddb:0-0 {event_type=RS_FLUSH_REGIONS, pid=39}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/7c7f87a6124bfe2f4049ee7803e0f98c/.tmp/A/a9b0a7760e764325a134f8bd6fd9c882 is 175, key is test_row_0/A:col10/1731751113405/Put/seqid=0 2024-11-16T09:58:34,929 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c7f87a6124bfe2f4049ee7803e0f98c, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T09:58:34,929 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52420 deadline: 1731751174928, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c7f87a6124bfe2f4049ee7803e0f98c, server=037c58a19ddb,37267,1731751077844 2024-11-16T09:58:34,929 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c7f87a6124bfe2f4049ee7803e0f98c, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T09:58:34,930 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52396 deadline: 1731751174929, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c7f87a6124bfe2f4049ee7803e0f98c, server=037c58a19ddb,37267,1731751077844 2024-11-16T09:58:34,930 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c7f87a6124bfe2f4049ee7803e0f98c, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T09:58:34,930 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52400 deadline: 1731751174929, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c7f87a6124bfe2f4049ee7803e0f98c, server=037c58a19ddb,37267,1731751077844 2024-11-16T09:58:34,930 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c7f87a6124bfe2f4049ee7803e0f98c, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T09:58:34,931 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731751108989.7c7f87a6124bfe2f4049ee7803e0f98c., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 , the old value is region=TestAcidGuarantees,,1731751108989.7c7f87a6124bfe2f4049ee7803e0f98c., hostname=037c58a19ddb,37267,1731751077844, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c7f87a6124bfe2f4049ee7803e0f98c, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:58:34,931 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52428 deadline: 1731751174929, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c7f87a6124bfe2f4049ee7803e0f98c, server=037c58a19ddb,37267,1731751077844 2024-11-16T09:58:34,931 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731751108989.7c7f87a6124bfe2f4049ee7803e0f98c., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c7f87a6124bfe2f4049ee7803e0f98c, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:58:34,931 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731751108989.7c7f87a6124bfe2f4049ee7803e0f98c., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 because the exception is null or not the one we care about 2024-11-16T09:58:34,931 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731751108989.7c7f87a6124bfe2f4049ee7803e0f98c., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 , the old value is region=TestAcidGuarantees,,1731751108989.7c7f87a6124bfe2f4049ee7803e0f98c., hostname=037c58a19ddb,37267,1731751077844, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c7f87a6124bfe2f4049ee7803e0f98c, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:58:34,931 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731751108989.7c7f87a6124bfe2f4049ee7803e0f98c., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 , the old value is region=TestAcidGuarantees,,1731751108989.7c7f87a6124bfe2f4049ee7803e0f98c., hostname=037c58a19ddb,37267,1731751077844, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c7f87a6124bfe2f4049ee7803e0f98c, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:58:34,931 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731751108989.7c7f87a6124bfe2f4049ee7803e0f98c., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c7f87a6124bfe2f4049ee7803e0f98c, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:58:34,931 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731751108989.7c7f87a6124bfe2f4049ee7803e0f98c., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c7f87a6124bfe2f4049ee7803e0f98c, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:58:34,931 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731751108989.7c7f87a6124bfe2f4049ee7803e0f98c., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 because the exception is null or not the one we care about 2024-11-16T09:58:34,931 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731751108989.7c7f87a6124bfe2f4049ee7803e0f98c., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 because the exception is null or not the one we care about 2024-11-16T09:58:34,931 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731751108989.7c7f87a6124bfe2f4049ee7803e0f98c., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 , the old value is region=TestAcidGuarantees,,1731751108989.7c7f87a6124bfe2f4049ee7803e0f98c., hostname=037c58a19ddb,37267,1731751077844, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c7f87a6124bfe2f4049ee7803e0f98c, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:58:34,932 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731751108989.7c7f87a6124bfe2f4049ee7803e0f98c., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c7f87a6124bfe2f4049ee7803e0f98c, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:58:34,932 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731751108989.7c7f87a6124bfe2f4049ee7803e0f98c., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 because the exception is null or not the one we care about 2024-11-16T09:58:34,940 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c7f87a6124bfe2f4049ee7803e0f98c, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T09:58:34,940 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52392 deadline: 1731751174933, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c7f87a6124bfe2f4049ee7803e0f98c, server=037c58a19ddb,37267,1731751077844 2024-11-16T09:58:34,940 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731751108989.7c7f87a6124bfe2f4049ee7803e0f98c., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 , the old value is region=TestAcidGuarantees,,1731751108989.7c7f87a6124bfe2f4049ee7803e0f98c., hostname=037c58a19ddb,37267,1731751077844, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c7f87a6124bfe2f4049ee7803e0f98c, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:58:34,940 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731751108989.7c7f87a6124bfe2f4049ee7803e0f98c., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c7f87a6124bfe2f4049ee7803e0f98c, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:58:34,941 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731751108989.7c7f87a6124bfe2f4049ee7803e0f98c., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 because the exception is null or not the one we care about 2024-11-16T09:58:34,943 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37591 is added to blk_1073741978_1154 (size=30955) 2024-11-16T09:58:34,945 INFO [RS_FLUSH_OPERATIONS-regionserver/037c58a19ddb:0-0 {event_type=RS_FLUSH_REGIONS, pid=39}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=53, memsize=20.1 K, hasBloomFilter=true, into tmp file hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/7c7f87a6124bfe2f4049ee7803e0f98c/.tmp/A/a9b0a7760e764325a134f8bd6fd9c882 2024-11-16T09:58:34,957 DEBUG [RS_FLUSH_OPERATIONS-regionserver/037c58a19ddb:0-0 {event_type=RS_FLUSH_REGIONS, pid=39}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/7c7f87a6124bfe2f4049ee7803e0f98c/.tmp/B/1dd7fb59b8be4b04ae256d3db0407388 is 50, key is test_row_0/B:col10/1731751113405/Put/seqid=0 2024-11-16T09:58:34,969 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37591 is added to blk_1073741979_1155 (size=12001) 2024-11-16T09:58:35,243 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c7f87a6124bfe2f4049ee7803e0f98c, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T09:58:35,244 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52396 deadline: 1731751175243, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c7f87a6124bfe2f4049ee7803e0f98c, server=037c58a19ddb,37267,1731751077844 2024-11-16T09:58:35,244 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c7f87a6124bfe2f4049ee7803e0f98c, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T09:58:35,244 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731751108989.7c7f87a6124bfe2f4049ee7803e0f98c., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 , the old value is region=TestAcidGuarantees,,1731751108989.7c7f87a6124bfe2f4049ee7803e0f98c., hostname=037c58a19ddb,37267,1731751077844, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c7f87a6124bfe2f4049ee7803e0f98c, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:58:35,244 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c7f87a6124bfe2f4049ee7803e0f98c, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T09:58:35,244 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731751108989.7c7f87a6124bfe2f4049ee7803e0f98c., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c7f87a6124bfe2f4049ee7803e0f98c, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:58:35,244 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52420 deadline: 1731751175243, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c7f87a6124bfe2f4049ee7803e0f98c, server=037c58a19ddb,37267,1731751077844 2024-11-16T09:58:35,244 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731751108989.7c7f87a6124bfe2f4049ee7803e0f98c., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 because the exception is null or not the one we care about 2024-11-16T09:58:35,244 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52400 deadline: 1731751175243, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c7f87a6124bfe2f4049ee7803e0f98c, server=037c58a19ddb,37267,1731751077844 2024-11-16T09:58:35,244 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c7f87a6124bfe2f4049ee7803e0f98c, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T09:58:35,245 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731751108989.7c7f87a6124bfe2f4049ee7803e0f98c., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 , the old value is region=TestAcidGuarantees,,1731751108989.7c7f87a6124bfe2f4049ee7803e0f98c., hostname=037c58a19ddb,37267,1731751077844, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c7f87a6124bfe2f4049ee7803e0f98c, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:58:35,245 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731751108989.7c7f87a6124bfe2f4049ee7803e0f98c., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 , the old value is region=TestAcidGuarantees,,1731751108989.7c7f87a6124bfe2f4049ee7803e0f98c., hostname=037c58a19ddb,37267,1731751077844, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c7f87a6124bfe2f4049ee7803e0f98c, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:58:35,245 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52428 deadline: 1731751175243, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c7f87a6124bfe2f4049ee7803e0f98c, server=037c58a19ddb,37267,1731751077844 2024-11-16T09:58:35,245 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731751108989.7c7f87a6124bfe2f4049ee7803e0f98c., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c7f87a6124bfe2f4049ee7803e0f98c, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:58:35,245 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731751108989.7c7f87a6124bfe2f4049ee7803e0f98c., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c7f87a6124bfe2f4049ee7803e0f98c, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:58:35,245 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731751108989.7c7f87a6124bfe2f4049ee7803e0f98c., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 because the exception is null or not the one we care about 2024-11-16T09:58:35,245 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731751108989.7c7f87a6124bfe2f4049ee7803e0f98c., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 because the exception is null or not the one we care about 2024-11-16T09:58:35,245 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731751108989.7c7f87a6124bfe2f4049ee7803e0f98c., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 , the old value is region=TestAcidGuarantees,,1731751108989.7c7f87a6124bfe2f4049ee7803e0f98c., hostname=037c58a19ddb,37267,1731751077844, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c7f87a6124bfe2f4049ee7803e0f98c, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:58:35,245 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731751108989.7c7f87a6124bfe2f4049ee7803e0f98c., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c7f87a6124bfe2f4049ee7803e0f98c, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:58:35,245 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731751108989.7c7f87a6124bfe2f4049ee7803e0f98c., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 because the exception is null or not the one we care about 2024-11-16T09:58:35,256 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c7f87a6124bfe2f4049ee7803e0f98c, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T09:58:35,256 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52392 deadline: 1731751175252, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c7f87a6124bfe2f4049ee7803e0f98c, server=037c58a19ddb,37267,1731751077844 2024-11-16T09:58:35,256 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731751108989.7c7f87a6124bfe2f4049ee7803e0f98c., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 , the old value is region=TestAcidGuarantees,,1731751108989.7c7f87a6124bfe2f4049ee7803e0f98c., hostname=037c58a19ddb,37267,1731751077844, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c7f87a6124bfe2f4049ee7803e0f98c, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:58:35,256 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731751108989.7c7f87a6124bfe2f4049ee7803e0f98c., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c7f87a6124bfe2f4049ee7803e0f98c, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:58:35,256 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731751108989.7c7f87a6124bfe2f4049ee7803e0f98c., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 because the exception is null or not the one we care about 2024-11-16T09:58:35,370 INFO [RS_FLUSH_OPERATIONS-regionserver/037c58a19ddb:0-0 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=53 (bloomFilter=true), to=hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/7c7f87a6124bfe2f4049ee7803e0f98c/.tmp/B/1dd7fb59b8be4b04ae256d3db0407388 2024-11-16T09:58:35,379 DEBUG [RS_FLUSH_OPERATIONS-regionserver/037c58a19ddb:0-0 {event_type=RS_FLUSH_REGIONS, pid=39}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/7c7f87a6124bfe2f4049ee7803e0f98c/.tmp/C/8ee49e7518574d3b98cb848b8d210202 is 50, key is test_row_0/C:col10/1731751113405/Put/seqid=0 2024-11-16T09:58:35,383 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37591 is added to blk_1073741980_1156 (size=12001) 2024-11-16T09:58:35,384 INFO [RS_FLUSH_OPERATIONS-regionserver/037c58a19ddb:0-0 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=53 (bloomFilter=true), to=hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/7c7f87a6124bfe2f4049ee7803e0f98c/.tmp/C/8ee49e7518574d3b98cb848b8d210202 2024-11-16T09:58:35,390 DEBUG [RS_FLUSH_OPERATIONS-regionserver/037c58a19ddb:0-0 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/7c7f87a6124bfe2f4049ee7803e0f98c/.tmp/A/a9b0a7760e764325a134f8bd6fd9c882 as hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/7c7f87a6124bfe2f4049ee7803e0f98c/A/a9b0a7760e764325a134f8bd6fd9c882 2024-11-16T09:58:35,394 INFO [RS_FLUSH_OPERATIONS-regionserver/037c58a19ddb:0-0 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/7c7f87a6124bfe2f4049ee7803e0f98c/A/a9b0a7760e764325a134f8bd6fd9c882, entries=150, sequenceid=53, filesize=30.2 K 2024-11-16T09:58:35,395 DEBUG [RS_FLUSH_OPERATIONS-regionserver/037c58a19ddb:0-0 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/7c7f87a6124bfe2f4049ee7803e0f98c/.tmp/B/1dd7fb59b8be4b04ae256d3db0407388 as hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/7c7f87a6124bfe2f4049ee7803e0f98c/B/1dd7fb59b8be4b04ae256d3db0407388 2024-11-16T09:58:35,401 INFO [RS_FLUSH_OPERATIONS-regionserver/037c58a19ddb:0-0 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/7c7f87a6124bfe2f4049ee7803e0f98c/B/1dd7fb59b8be4b04ae256d3db0407388, entries=150, sequenceid=53, filesize=11.7 K 2024-11-16T09:58:35,402 DEBUG [RS_FLUSH_OPERATIONS-regionserver/037c58a19ddb:0-0 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/7c7f87a6124bfe2f4049ee7803e0f98c/.tmp/C/8ee49e7518574d3b98cb848b8d210202 as hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/7c7f87a6124bfe2f4049ee7803e0f98c/C/8ee49e7518574d3b98cb848b8d210202 2024-11-16T09:58:35,406 INFO [RS_FLUSH_OPERATIONS-regionserver/037c58a19ddb:0-0 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/7c7f87a6124bfe2f4049ee7803e0f98c/C/8ee49e7518574d3b98cb848b8d210202, entries=150, sequenceid=53, filesize=11.7 K 2024-11-16T09:58:35,407 INFO [RS_FLUSH_OPERATIONS-regionserver/037c58a19ddb:0-0 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~158.91 KB/162720, currentSize=147.60 KB/151140 for 7c7f87a6124bfe2f4049ee7803e0f98c in 907ms, sequenceid=53, compaction requested=true 2024-11-16T09:58:35,407 DEBUG [RS_FLUSH_OPERATIONS-regionserver/037c58a19ddb:0-0 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.HRegion(2603): Flush status journal for 7c7f87a6124bfe2f4049ee7803e0f98c: 2024-11-16T09:58:35,407 DEBUG [RS_FLUSH_OPERATIONS-regionserver/037c58a19ddb:0-0 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731751108989.7c7f87a6124bfe2f4049ee7803e0f98c. 2024-11-16T09:58:35,407 DEBUG [RS_FLUSH_OPERATIONS-regionserver/037c58a19ddb:0-0 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=39 2024-11-16T09:58:35,408 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37877 {}] master.HMaster(4169): Remote procedure done, pid=39 2024-11-16T09:58:35,412 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=39, resume processing ppid=38 2024-11-16T09:58:35,412 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=39, ppid=38, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 3.2440 sec 2024-11-16T09:58:35,414 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=38, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=38, table=TestAcidGuarantees in 3.2560 sec 2024-11-16T09:58:35,755 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] regionserver.HRegion(8855): Flush requested on 7c7f87a6124bfe2f4049ee7803e0f98c 2024-11-16T09:58:35,755 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-16T09:58:35,755 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 7c7f87a6124bfe2f4049ee7803e0f98c 3/3 column families, dataSize=154.31 KB heapSize=405.05 KB 2024-11-16T09:58:35,764 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241116e038e92c50cf48e597852abdb185a5c2_7c7f87a6124bfe2f4049ee7803e0f98c is 50, key is test_row_0/A:col10/1731751114601/Put/seqid=0 2024-11-16T09:58:35,766 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c7f87a6124bfe2f4049ee7803e0f98c, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T09:58:35,766 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c7f87a6124bfe2f4049ee7803e0f98c, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T09:58:35,767 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52420 deadline: 1731751175763, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c7f87a6124bfe2f4049ee7803e0f98c, server=037c58a19ddb,37267,1731751077844 2024-11-16T09:58:35,767 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52392 deadline: 1731751175763, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c7f87a6124bfe2f4049ee7803e0f98c, server=037c58a19ddb,37267,1731751077844 2024-11-16T09:58:35,767 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731751108989.7c7f87a6124bfe2f4049ee7803e0f98c., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 , the old value is region=TestAcidGuarantees,,1731751108989.7c7f87a6124bfe2f4049ee7803e0f98c., hostname=037c58a19ddb,37267,1731751077844, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c7f87a6124bfe2f4049ee7803e0f98c, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:58:35,767 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731751108989.7c7f87a6124bfe2f4049ee7803e0f98c., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c7f87a6124bfe2f4049ee7803e0f98c, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:58:35,767 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731751108989.7c7f87a6124bfe2f4049ee7803e0f98c., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 , the old value is region=TestAcidGuarantees,,1731751108989.7c7f87a6124bfe2f4049ee7803e0f98c., hostname=037c58a19ddb,37267,1731751077844, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c7f87a6124bfe2f4049ee7803e0f98c, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:58:35,767 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731751108989.7c7f87a6124bfe2f4049ee7803e0f98c., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 because the exception is null or not the one we care about 2024-11-16T09:58:35,767 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731751108989.7c7f87a6124bfe2f4049ee7803e0f98c., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c7f87a6124bfe2f4049ee7803e0f98c, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:58:35,767 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731751108989.7c7f87a6124bfe2f4049ee7803e0f98c., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 because the exception is null or not the one we care about 2024-11-16T09:58:35,772 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c7f87a6124bfe2f4049ee7803e0f98c, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T09:58:35,772 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52400 deadline: 1731751175765, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c7f87a6124bfe2f4049ee7803e0f98c, server=037c58a19ddb,37267,1731751077844 2024-11-16T09:58:35,772 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731751108989.7c7f87a6124bfe2f4049ee7803e0f98c., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 , the old value is region=TestAcidGuarantees,,1731751108989.7c7f87a6124bfe2f4049ee7803e0f98c., hostname=037c58a19ddb,37267,1731751077844, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c7f87a6124bfe2f4049ee7803e0f98c, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:58:35,772 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c7f87a6124bfe2f4049ee7803e0f98c, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T09:58:35,772 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731751108989.7c7f87a6124bfe2f4049ee7803e0f98c., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c7f87a6124bfe2f4049ee7803e0f98c, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:58:35,773 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731751108989.7c7f87a6124bfe2f4049ee7803e0f98c., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 because the exception is null or not the one we care about 2024-11-16T09:58:35,773 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52428 deadline: 1731751175767, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c7f87a6124bfe2f4049ee7803e0f98c, server=037c58a19ddb,37267,1731751077844 2024-11-16T09:58:35,773 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731751108989.7c7f87a6124bfe2f4049ee7803e0f98c., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 , the old value is region=TestAcidGuarantees,,1731751108989.7c7f87a6124bfe2f4049ee7803e0f98c., hostname=037c58a19ddb,37267,1731751077844, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c7f87a6124bfe2f4049ee7803e0f98c, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:58:35,773 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c7f87a6124bfe2f4049ee7803e0f98c, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T09:58:35,773 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731751108989.7c7f87a6124bfe2f4049ee7803e0f98c., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c7f87a6124bfe2f4049ee7803e0f98c, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:58:35,773 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731751108989.7c7f87a6124bfe2f4049ee7803e0f98c., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 because the exception is null or not the one we care about 2024-11-16T09:58:35,773 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52396 deadline: 1731751175767, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c7f87a6124bfe2f4049ee7803e0f98c, server=037c58a19ddb,37267,1731751077844 2024-11-16T09:58:35,773 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731751108989.7c7f87a6124bfe2f4049ee7803e0f98c., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 , the old value is region=TestAcidGuarantees,,1731751108989.7c7f87a6124bfe2f4049ee7803e0f98c., hostname=037c58a19ddb,37267,1731751077844, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c7f87a6124bfe2f4049ee7803e0f98c, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:58:35,773 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731751108989.7c7f87a6124bfe2f4049ee7803e0f98c., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c7f87a6124bfe2f4049ee7803e0f98c, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:58:35,773 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731751108989.7c7f87a6124bfe2f4049ee7803e0f98c., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 because the exception is null or not the one we care about 2024-11-16T09:58:35,778 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37591 is added to blk_1073741981_1157 (size=14594) 2024-11-16T09:58:35,779 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T09:58:35,793 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241116e038e92c50cf48e597852abdb185a5c2_7c7f87a6124bfe2f4049ee7803e0f98c to hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241116e038e92c50cf48e597852abdb185a5c2_7c7f87a6124bfe2f4049ee7803e0f98c 2024-11-16T09:58:35,794 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/7c7f87a6124bfe2f4049ee7803e0f98c/.tmp/A/8aaa28ea93ee462eaa017eee7d302c00, store: [table=TestAcidGuarantees family=A region=7c7f87a6124bfe2f4049ee7803e0f98c] 2024-11-16T09:58:35,795 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/7c7f87a6124bfe2f4049ee7803e0f98c/.tmp/A/8aaa28ea93ee462eaa017eee7d302c00 is 175, key is test_row_0/A:col10/1731751114601/Put/seqid=0 2024-11-16T09:58:35,816 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37591 is added to blk_1073741982_1158 (size=39549) 2024-11-16T09:58:35,875 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c7f87a6124bfe2f4049ee7803e0f98c, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T09:58:35,875 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52420 deadline: 1731751175872, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c7f87a6124bfe2f4049ee7803e0f98c, server=037c58a19ddb,37267,1731751077844 2024-11-16T09:58:35,876 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731751108989.7c7f87a6124bfe2f4049ee7803e0f98c., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 , the old value is region=TestAcidGuarantees,,1731751108989.7c7f87a6124bfe2f4049ee7803e0f98c., hostname=037c58a19ddb,37267,1731751077844, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c7f87a6124bfe2f4049ee7803e0f98c, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:58:35,876 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731751108989.7c7f87a6124bfe2f4049ee7803e0f98c., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c7f87a6124bfe2f4049ee7803e0f98c, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:58:35,876 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731751108989.7c7f87a6124bfe2f4049ee7803e0f98c., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 because the exception is null or not the one we care about 2024-11-16T09:58:35,886 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c7f87a6124bfe2f4049ee7803e0f98c, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T09:58:35,886 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52400 deadline: 1731751175883, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c7f87a6124bfe2f4049ee7803e0f98c, server=037c58a19ddb,37267,1731751077844 2024-11-16T09:58:35,887 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c7f87a6124bfe2f4049ee7803e0f98c, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T09:58:35,887 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731751108989.7c7f87a6124bfe2f4049ee7803e0f98c., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 , the old value is region=TestAcidGuarantees,,1731751108989.7c7f87a6124bfe2f4049ee7803e0f98c., hostname=037c58a19ddb,37267,1731751077844, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c7f87a6124bfe2f4049ee7803e0f98c, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:58:35,887 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731751108989.7c7f87a6124bfe2f4049ee7803e0f98c., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c7f87a6124bfe2f4049ee7803e0f98c, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:58:35,887 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52428 deadline: 1731751175883, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c7f87a6124bfe2f4049ee7803e0f98c, server=037c58a19ddb,37267,1731751077844 2024-11-16T09:58:35,887 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731751108989.7c7f87a6124bfe2f4049ee7803e0f98c., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 because the exception is null or not the one we care about 2024-11-16T09:58:35,887 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731751108989.7c7f87a6124bfe2f4049ee7803e0f98c., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 , the old value is region=TestAcidGuarantees,,1731751108989.7c7f87a6124bfe2f4049ee7803e0f98c., hostname=037c58a19ddb,37267,1731751077844, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c7f87a6124bfe2f4049ee7803e0f98c, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:58:35,887 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731751108989.7c7f87a6124bfe2f4049ee7803e0f98c., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c7f87a6124bfe2f4049ee7803e0f98c, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:58:35,887 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c7f87a6124bfe2f4049ee7803e0f98c, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T09:58:35,887 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731751108989.7c7f87a6124bfe2f4049ee7803e0f98c., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 because the exception is null or not the one we care about 2024-11-16T09:58:35,887 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52396 deadline: 1731751175883, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c7f87a6124bfe2f4049ee7803e0f98c, server=037c58a19ddb,37267,1731751077844 2024-11-16T09:58:35,888 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731751108989.7c7f87a6124bfe2f4049ee7803e0f98c., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 , the old value is region=TestAcidGuarantees,,1731751108989.7c7f87a6124bfe2f4049ee7803e0f98c., hostname=037c58a19ddb,37267,1731751077844, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c7f87a6124bfe2f4049ee7803e0f98c, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:58:35,888 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731751108989.7c7f87a6124bfe2f4049ee7803e0f98c., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c7f87a6124bfe2f4049ee7803e0f98c, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:58:35,888 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731751108989.7c7f87a6124bfe2f4049ee7803e0f98c., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 because the exception is null or not the one we care about 2024-11-16T09:58:36,083 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c7f87a6124bfe2f4049ee7803e0f98c, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T09:58:36,084 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52420 deadline: 1731751176083, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c7f87a6124bfe2f4049ee7803e0f98c, server=037c58a19ddb,37267,1731751077844 2024-11-16T09:58:36,084 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731751108989.7c7f87a6124bfe2f4049ee7803e0f98c., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 , the old value is region=TestAcidGuarantees,,1731751108989.7c7f87a6124bfe2f4049ee7803e0f98c., hostname=037c58a19ddb,37267,1731751077844, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c7f87a6124bfe2f4049ee7803e0f98c, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:58:36,084 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731751108989.7c7f87a6124bfe2f4049ee7803e0f98c., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c7f87a6124bfe2f4049ee7803e0f98c, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:58:36,084 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731751108989.7c7f87a6124bfe2f4049ee7803e0f98c., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 because the exception is null or not the one we care about 2024-11-16T09:58:36,094 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c7f87a6124bfe2f4049ee7803e0f98c, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T09:58:36,094 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52400 deadline: 1731751176093, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c7f87a6124bfe2f4049ee7803e0f98c, server=037c58a19ddb,37267,1731751077844 2024-11-16T09:58:36,094 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c7f87a6124bfe2f4049ee7803e0f98c, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T09:58:36,094 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731751108989.7c7f87a6124bfe2f4049ee7803e0f98c., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 , the old value is region=TestAcidGuarantees,,1731751108989.7c7f87a6124bfe2f4049ee7803e0f98c., hostname=037c58a19ddb,37267,1731751077844, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c7f87a6124bfe2f4049ee7803e0f98c, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:58:36,094 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52428 deadline: 1731751176093, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c7f87a6124bfe2f4049ee7803e0f98c, server=037c58a19ddb,37267,1731751077844 2024-11-16T09:58:36,094 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731751108989.7c7f87a6124bfe2f4049ee7803e0f98c., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c7f87a6124bfe2f4049ee7803e0f98c, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:58:36,094 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c7f87a6124bfe2f4049ee7803e0f98c, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T09:58:36,094 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731751108989.7c7f87a6124bfe2f4049ee7803e0f98c., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 because the exception is null or not the one we care about 2024-11-16T09:58:36,095 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52396 deadline: 1731751176093, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c7f87a6124bfe2f4049ee7803e0f98c, server=037c58a19ddb,37267,1731751077844 2024-11-16T09:58:36,095 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731751108989.7c7f87a6124bfe2f4049ee7803e0f98c., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 , the old value is region=TestAcidGuarantees,,1731751108989.7c7f87a6124bfe2f4049ee7803e0f98c., hostname=037c58a19ddb,37267,1731751077844, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c7f87a6124bfe2f4049ee7803e0f98c, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:58:36,095 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731751108989.7c7f87a6124bfe2f4049ee7803e0f98c., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c7f87a6124bfe2f4049ee7803e0f98c, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:58:36,095 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731751108989.7c7f87a6124bfe2f4049ee7803e0f98c., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 because the exception is null or not the one we care about 2024-11-16T09:58:36,095 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731751108989.7c7f87a6124bfe2f4049ee7803e0f98c., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 , the old value is region=TestAcidGuarantees,,1731751108989.7c7f87a6124bfe2f4049ee7803e0f98c., hostname=037c58a19ddb,37267,1731751077844, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c7f87a6124bfe2f4049ee7803e0f98c, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:58:36,095 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731751108989.7c7f87a6124bfe2f4049ee7803e0f98c., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c7f87a6124bfe2f4049ee7803e0f98c, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:58:36,095 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731751108989.7c7f87a6124bfe2f4049ee7803e0f98c., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 because the exception is null or not the one we care about 2024-11-16T09:58:36,218 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=79, memsize=51.4 K, hasBloomFilter=true, into tmp file hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/7c7f87a6124bfe2f4049ee7803e0f98c/.tmp/A/8aaa28ea93ee462eaa017eee7d302c00 2024-11-16T09:58:36,228 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/7c7f87a6124bfe2f4049ee7803e0f98c/.tmp/B/b4b46e5bd82a454d819133a11c7a80cb is 50, key is test_row_0/B:col10/1731751114601/Put/seqid=0 2024-11-16T09:58:36,263 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37591 is added to blk_1073741983_1159 (size=12001) 2024-11-16T09:58:36,323 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37877 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=38 2024-11-16T09:58:36,323 INFO [RPCClient-NioEventLoopGroup-4-7 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-16T09:58:36,324 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37877 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-16T09:58:36,326 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37877 {}] procedure2.ProcedureExecutor(1139): Stored pid=40, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=40, table=TestAcidGuarantees 2024-11-16T09:58:36,327 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=40, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=40, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-16T09:58:36,328 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=40, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=40, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-16T09:58:36,328 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=41, ppid=40, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-16T09:58:36,329 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37877 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=40 2024-11-16T09:58:36,396 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c7f87a6124bfe2f4049ee7803e0f98c, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T09:58:36,396 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52420 deadline: 1731751176393, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c7f87a6124bfe2f4049ee7803e0f98c, server=037c58a19ddb,37267,1731751077844 2024-11-16T09:58:36,397 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731751108989.7c7f87a6124bfe2f4049ee7803e0f98c., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 , the old value is region=TestAcidGuarantees,,1731751108989.7c7f87a6124bfe2f4049ee7803e0f98c., hostname=037c58a19ddb,37267,1731751077844, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c7f87a6124bfe2f4049ee7803e0f98c, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:58:36,397 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731751108989.7c7f87a6124bfe2f4049ee7803e0f98c., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c7f87a6124bfe2f4049ee7803e0f98c, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:58:36,397 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731751108989.7c7f87a6124bfe2f4049ee7803e0f98c., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 because the exception is null or not the one we care about 2024-11-16T09:58:36,408 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c7f87a6124bfe2f4049ee7803e0f98c, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T09:58:36,408 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52400 deadline: 1731751176403, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c7f87a6124bfe2f4049ee7803e0f98c, server=037c58a19ddb,37267,1731751077844 2024-11-16T09:58:36,408 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c7f87a6124bfe2f4049ee7803e0f98c, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T09:58:36,409 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52428 deadline: 1731751176403, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c7f87a6124bfe2f4049ee7803e0f98c, server=037c58a19ddb,37267,1731751077844 2024-11-16T09:58:36,409 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c7f87a6124bfe2f4049ee7803e0f98c, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T09:58:36,409 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52396 deadline: 1731751176403, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c7f87a6124bfe2f4049ee7803e0f98c, server=037c58a19ddb,37267,1731751077844 2024-11-16T09:58:36,409 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731751108989.7c7f87a6124bfe2f4049ee7803e0f98c., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 , the old value is region=TestAcidGuarantees,,1731751108989.7c7f87a6124bfe2f4049ee7803e0f98c., hostname=037c58a19ddb,37267,1731751077844, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c7f87a6124bfe2f4049ee7803e0f98c, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:58:36,409 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731751108989.7c7f87a6124bfe2f4049ee7803e0f98c., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c7f87a6124bfe2f4049ee7803e0f98c, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:58:36,409 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731751108989.7c7f87a6124bfe2f4049ee7803e0f98c., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 because the exception is null or not the one we care about 2024-11-16T09:58:36,409 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731751108989.7c7f87a6124bfe2f4049ee7803e0f98c., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 , the old value is region=TestAcidGuarantees,,1731751108989.7c7f87a6124bfe2f4049ee7803e0f98c., hostname=037c58a19ddb,37267,1731751077844, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c7f87a6124bfe2f4049ee7803e0f98c, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:58:36,409 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731751108989.7c7f87a6124bfe2f4049ee7803e0f98c., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c7f87a6124bfe2f4049ee7803e0f98c, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:58:36,409 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731751108989.7c7f87a6124bfe2f4049ee7803e0f98c., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 because the exception is null or not the one we care about 2024-11-16T09:58:36,410 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731751108989.7c7f87a6124bfe2f4049ee7803e0f98c., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 , the old value is region=TestAcidGuarantees,,1731751108989.7c7f87a6124bfe2f4049ee7803e0f98c., hostname=037c58a19ddb,37267,1731751077844, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c7f87a6124bfe2f4049ee7803e0f98c, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:58:36,410 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731751108989.7c7f87a6124bfe2f4049ee7803e0f98c., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c7f87a6124bfe2f4049ee7803e0f98c, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:58:36,410 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731751108989.7c7f87a6124bfe2f4049ee7803e0f98c., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 because the exception is null or not the one we care about 2024-11-16T09:58:36,433 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37877 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=40 2024-11-16T09:58:36,482 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=37267 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=41 2024-11-16T09:58:36,482 DEBUG [RS_FLUSH_OPERATIONS-regionserver/037c58a19ddb:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731751108989.7c7f87a6124bfe2f4049ee7803e0f98c. 2024-11-16T09:58:36,482 DEBUG [RS_FLUSH_OPERATIONS-regionserver/037c58a19ddb:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731751108989.7c7f87a6124bfe2f4049ee7803e0f98c. as already flushing 2024-11-16T09:58:36,482 DEBUG [RS_FLUSH_OPERATIONS-regionserver/037c58a19ddb:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731751108989.7c7f87a6124bfe2f4049ee7803e0f98c. 2024-11-16T09:58:36,483 ERROR [RS_FLUSH_OPERATIONS-regionserver/037c58a19ddb:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] handler.RSProcedureHandler(58): pid=41 java.io.IOException: Unable to complete flush {ENCODED => 7c7f87a6124bfe2f4049ee7803e0f98c, NAME => 'TestAcidGuarantees,,1731751108989.7c7f87a6124bfe2f4049ee7803e0f98c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T09:58:36,483 DEBUG [RS_FLUSH_OPERATIONS-regionserver/037c58a19ddb:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=41 java.io.IOException: Unable to complete flush {ENCODED => 7c7f87a6124bfe2f4049ee7803e0f98c, NAME => 'TestAcidGuarantees,,1731751108989.7c7f87a6124bfe2f4049ee7803e0f98c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T09:58:36,484 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37877 {}] master.HMaster(4177): Remote procedure failed, pid=41 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 7c7f87a6124bfe2f4049ee7803e0f98c, NAME => 'TestAcidGuarantees,,1731751108989.7c7f87a6124bfe2f4049ee7803e0f98c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 7c7f87a6124bfe2f4049ee7803e0f98c, NAME => 'TestAcidGuarantees,,1731751108989.7c7f87a6124bfe2f4049ee7803e0f98c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T09:58:36,637 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=37267 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=41 2024-11-16T09:58:36,637 DEBUG [RS_FLUSH_OPERATIONS-regionserver/037c58a19ddb:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731751108989.7c7f87a6124bfe2f4049ee7803e0f98c. 2024-11-16T09:58:36,637 DEBUG [RS_FLUSH_OPERATIONS-regionserver/037c58a19ddb:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731751108989.7c7f87a6124bfe2f4049ee7803e0f98c. as already flushing 2024-11-16T09:58:36,637 DEBUG [RS_FLUSH_OPERATIONS-regionserver/037c58a19ddb:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731751108989.7c7f87a6124bfe2f4049ee7803e0f98c. 2024-11-16T09:58:36,638 ERROR [RS_FLUSH_OPERATIONS-regionserver/037c58a19ddb:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] handler.RSProcedureHandler(58): pid=41 java.io.IOException: Unable to complete flush {ENCODED => 7c7f87a6124bfe2f4049ee7803e0f98c, NAME => 'TestAcidGuarantees,,1731751108989.7c7f87a6124bfe2f4049ee7803e0f98c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T09:58:36,638 DEBUG [RS_FLUSH_OPERATIONS-regionserver/037c58a19ddb:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=41 java.io.IOException: Unable to complete flush {ENCODED => 7c7f87a6124bfe2f4049ee7803e0f98c, NAME => 'TestAcidGuarantees,,1731751108989.7c7f87a6124bfe2f4049ee7803e0f98c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T09:58:36,639 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37877 {}] master.HMaster(4177): Remote procedure failed, pid=41 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 7c7f87a6124bfe2f4049ee7803e0f98c, NAME => 'TestAcidGuarantees,,1731751108989.7c7f87a6124bfe2f4049ee7803e0f98c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 7c7f87a6124bfe2f4049ee7803e0f98c, NAME => 'TestAcidGuarantees,,1731751108989.7c7f87a6124bfe2f4049ee7803e0f98c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T09:58:36,643 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37877 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=40 2024-11-16T09:58:36,663 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=51.44 KB at sequenceid=79 (bloomFilter=true), to=hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/7c7f87a6124bfe2f4049ee7803e0f98c/.tmp/B/b4b46e5bd82a454d819133a11c7a80cb 2024-11-16T09:58:36,677 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/7c7f87a6124bfe2f4049ee7803e0f98c/.tmp/C/39d84edc31624bc4bca34f4812ae659d is 50, key is test_row_0/C:col10/1731751114601/Put/seqid=0 2024-11-16T09:58:36,720 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37591 is added to blk_1073741984_1160 (size=12001) 2024-11-16T09:58:36,777 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c7f87a6124bfe2f4049ee7803e0f98c, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T09:58:36,777 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52392 deadline: 1731751176773, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c7f87a6124bfe2f4049ee7803e0f98c, server=037c58a19ddb,37267,1731751077844 2024-11-16T09:58:36,777 WARN [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRpcRetryingCaller(177): Call to 037c58a19ddb,37267,1731751077844 for 'test_row_2' in 7c7f87a6124bfe2f4049ee7803e0f98c of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2174 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c7f87a6124bfe2f4049ee7803e0f98c, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor44.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c7f87a6124bfe2f4049ee7803e0f98c, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-16T09:58:36,778 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731751108989.7c7f87a6124bfe2f4049ee7803e0f98c., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 , the old value is region=TestAcidGuarantees,,1731751108989.7c7f87a6124bfe2f4049ee7803e0f98c., hostname=037c58a19ddb,37267,1731751077844, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c7f87a6124bfe2f4049ee7803e0f98c, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:58:36,778 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731751108989.7c7f87a6124bfe2f4049ee7803e0f98c., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c7f87a6124bfe2f4049ee7803e0f98c, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:58:36,778 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731751108989.7c7f87a6124bfe2f4049ee7803e0f98c., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 because the exception is null or not the one we care about 2024-11-16T09:58:36,793 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=37267 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=41 2024-11-16T09:58:36,793 DEBUG [RS_FLUSH_OPERATIONS-regionserver/037c58a19ddb:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731751108989.7c7f87a6124bfe2f4049ee7803e0f98c. 2024-11-16T09:58:36,793 DEBUG [RS_FLUSH_OPERATIONS-regionserver/037c58a19ddb:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731751108989.7c7f87a6124bfe2f4049ee7803e0f98c. as already flushing 2024-11-16T09:58:36,793 DEBUG [RS_FLUSH_OPERATIONS-regionserver/037c58a19ddb:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731751108989.7c7f87a6124bfe2f4049ee7803e0f98c. 2024-11-16T09:58:36,794 ERROR [RS_FLUSH_OPERATIONS-regionserver/037c58a19ddb:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] handler.RSProcedureHandler(58): pid=41 java.io.IOException: Unable to complete flush {ENCODED => 7c7f87a6124bfe2f4049ee7803e0f98c, NAME => 'TestAcidGuarantees,,1731751108989.7c7f87a6124bfe2f4049ee7803e0f98c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T09:58:36,794 DEBUG [RS_FLUSH_OPERATIONS-regionserver/037c58a19ddb:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=41 java.io.IOException: Unable to complete flush {ENCODED => 7c7f87a6124bfe2f4049ee7803e0f98c, NAME => 'TestAcidGuarantees,,1731751108989.7c7f87a6124bfe2f4049ee7803e0f98c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T09:58:36,794 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37877 {}] master.HMaster(4177): Remote procedure failed, pid=41 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 7c7f87a6124bfe2f4049ee7803e0f98c, NAME => 'TestAcidGuarantees,,1731751108989.7c7f87a6124bfe2f4049ee7803e0f98c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 7c7f87a6124bfe2f4049ee7803e0f98c, NAME => 'TestAcidGuarantees,,1731751108989.7c7f87a6124bfe2f4049ee7803e0f98c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T09:58:36,906 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c7f87a6124bfe2f4049ee7803e0f98c, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T09:58:36,906 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52420 deadline: 1731751176903, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c7f87a6124bfe2f4049ee7803e0f98c, server=037c58a19ddb,37267,1731751077844 2024-11-16T09:58:36,906 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731751108989.7c7f87a6124bfe2f4049ee7803e0f98c., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 , the old value is region=TestAcidGuarantees,,1731751108989.7c7f87a6124bfe2f4049ee7803e0f98c., hostname=037c58a19ddb,37267,1731751077844, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c7f87a6124bfe2f4049ee7803e0f98c, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:58:36,907 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731751108989.7c7f87a6124bfe2f4049ee7803e0f98c., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c7f87a6124bfe2f4049ee7803e0f98c, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:58:36,907 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731751108989.7c7f87a6124bfe2f4049ee7803e0f98c., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 because the exception is null or not the one we care about 2024-11-16T09:58:36,917 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c7f87a6124bfe2f4049ee7803e0f98c, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T09:58:36,917 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52428 deadline: 1731751176913, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c7f87a6124bfe2f4049ee7803e0f98c, server=037c58a19ddb,37267,1731751077844 2024-11-16T09:58:36,918 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731751108989.7c7f87a6124bfe2f4049ee7803e0f98c., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 , the old value is region=TestAcidGuarantees,,1731751108989.7c7f87a6124bfe2f4049ee7803e0f98c., hostname=037c58a19ddb,37267,1731751077844, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c7f87a6124bfe2f4049ee7803e0f98c, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:58:36,918 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731751108989.7c7f87a6124bfe2f4049ee7803e0f98c., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c7f87a6124bfe2f4049ee7803e0f98c, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:58:36,918 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731751108989.7c7f87a6124bfe2f4049ee7803e0f98c., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 because the exception is null or not the one we care about 2024-11-16T09:58:36,924 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c7f87a6124bfe2f4049ee7803e0f98c, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T09:58:36,925 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52400 deadline: 1731751176923, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c7f87a6124bfe2f4049ee7803e0f98c, server=037c58a19ddb,37267,1731751077844 2024-11-16T09:58:36,925 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731751108989.7c7f87a6124bfe2f4049ee7803e0f98c., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 , the old value is region=TestAcidGuarantees,,1731751108989.7c7f87a6124bfe2f4049ee7803e0f98c., hostname=037c58a19ddb,37267,1731751077844, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c7f87a6124bfe2f4049ee7803e0f98c, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:58:36,925 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c7f87a6124bfe2f4049ee7803e0f98c, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T09:58:36,925 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731751108989.7c7f87a6124bfe2f4049ee7803e0f98c., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c7f87a6124bfe2f4049ee7803e0f98c, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:58:36,925 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731751108989.7c7f87a6124bfe2f4049ee7803e0f98c., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 because the exception is null or not the one we care about 2024-11-16T09:58:36,925 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52396 deadline: 1731751176924, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c7f87a6124bfe2f4049ee7803e0f98c, server=037c58a19ddb,37267,1731751077844 2024-11-16T09:58:36,926 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731751108989.7c7f87a6124bfe2f4049ee7803e0f98c., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 , the old value is region=TestAcidGuarantees,,1731751108989.7c7f87a6124bfe2f4049ee7803e0f98c., hostname=037c58a19ddb,37267,1731751077844, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c7f87a6124bfe2f4049ee7803e0f98c, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:58:36,926 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731751108989.7c7f87a6124bfe2f4049ee7803e0f98c., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c7f87a6124bfe2f4049ee7803e0f98c, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:58:36,926 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731751108989.7c7f87a6124bfe2f4049ee7803e0f98c., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 because the exception is null or not the one we care about 2024-11-16T09:58:36,948 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=37267 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=41 2024-11-16T09:58:36,948 DEBUG [RS_FLUSH_OPERATIONS-regionserver/037c58a19ddb:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731751108989.7c7f87a6124bfe2f4049ee7803e0f98c. 2024-11-16T09:58:36,948 DEBUG [RS_FLUSH_OPERATIONS-regionserver/037c58a19ddb:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731751108989.7c7f87a6124bfe2f4049ee7803e0f98c. as already flushing 2024-11-16T09:58:36,949 DEBUG [RS_FLUSH_OPERATIONS-regionserver/037c58a19ddb:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731751108989.7c7f87a6124bfe2f4049ee7803e0f98c. 2024-11-16T09:58:36,949 ERROR [RS_FLUSH_OPERATIONS-regionserver/037c58a19ddb:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] handler.RSProcedureHandler(58): pid=41 java.io.IOException: Unable to complete flush {ENCODED => 7c7f87a6124bfe2f4049ee7803e0f98c, NAME => 'TestAcidGuarantees,,1731751108989.7c7f87a6124bfe2f4049ee7803e0f98c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T09:58:36,949 DEBUG [RS_FLUSH_OPERATIONS-regionserver/037c58a19ddb:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=41 java.io.IOException: Unable to complete flush {ENCODED => 7c7f87a6124bfe2f4049ee7803e0f98c, NAME => 'TestAcidGuarantees,,1731751108989.7c7f87a6124bfe2f4049ee7803e0f98c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T09:58:36,950 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37877 {}] master.HMaster(4177): Remote procedure failed, pid=41 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 7c7f87a6124bfe2f4049ee7803e0f98c, NAME => 'TestAcidGuarantees,,1731751108989.7c7f87a6124bfe2f4049ee7803e0f98c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 7c7f87a6124bfe2f4049ee7803e0f98c, NAME => 'TestAcidGuarantees,,1731751108989.7c7f87a6124bfe2f4049ee7803e0f98c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T09:58:36,953 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37877 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=40 2024-11-16T09:58:37,103 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=37267 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=41 2024-11-16T09:58:37,103 DEBUG [RS_FLUSH_OPERATIONS-regionserver/037c58a19ddb:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731751108989.7c7f87a6124bfe2f4049ee7803e0f98c. 2024-11-16T09:58:37,104 DEBUG [RS_FLUSH_OPERATIONS-regionserver/037c58a19ddb:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731751108989.7c7f87a6124bfe2f4049ee7803e0f98c. as already flushing 2024-11-16T09:58:37,104 DEBUG [RS_FLUSH_OPERATIONS-regionserver/037c58a19ddb:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731751108989.7c7f87a6124bfe2f4049ee7803e0f98c. 2024-11-16T09:58:37,104 ERROR [RS_FLUSH_OPERATIONS-regionserver/037c58a19ddb:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] handler.RSProcedureHandler(58): pid=41 java.io.IOException: Unable to complete flush {ENCODED => 7c7f87a6124bfe2f4049ee7803e0f98c, NAME => 'TestAcidGuarantees,,1731751108989.7c7f87a6124bfe2f4049ee7803e0f98c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T09:58:37,104 DEBUG [RS_FLUSH_OPERATIONS-regionserver/037c58a19ddb:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=41 java.io.IOException: Unable to complete flush {ENCODED => 7c7f87a6124bfe2f4049ee7803e0f98c, NAME => 'TestAcidGuarantees,,1731751108989.7c7f87a6124bfe2f4049ee7803e0f98c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T09:58:37,105 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37877 {}] master.HMaster(4177): Remote procedure failed, pid=41 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 7c7f87a6124bfe2f4049ee7803e0f98c, NAME => 'TestAcidGuarantees,,1731751108989.7c7f87a6124bfe2f4049ee7803e0f98c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 7c7f87a6124bfe2f4049ee7803e0f98c, NAME => 'TestAcidGuarantees,,1731751108989.7c7f87a6124bfe2f4049ee7803e0f98c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T09:58:37,129 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=51.44 KB at sequenceid=79 (bloomFilter=true), to=hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/7c7f87a6124bfe2f4049ee7803e0f98c/.tmp/C/39d84edc31624bc4bca34f4812ae659d 2024-11-16T09:58:37,138 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/7c7f87a6124bfe2f4049ee7803e0f98c/.tmp/A/8aaa28ea93ee462eaa017eee7d302c00 as hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/7c7f87a6124bfe2f4049ee7803e0f98c/A/8aaa28ea93ee462eaa017eee7d302c00 2024-11-16T09:58:37,146 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/7c7f87a6124bfe2f4049ee7803e0f98c/A/8aaa28ea93ee462eaa017eee7d302c00, entries=200, sequenceid=79, filesize=38.6 K 2024-11-16T09:58:37,148 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/7c7f87a6124bfe2f4049ee7803e0f98c/.tmp/B/b4b46e5bd82a454d819133a11c7a80cb as hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/7c7f87a6124bfe2f4049ee7803e0f98c/B/b4b46e5bd82a454d819133a11c7a80cb 2024-11-16T09:58:37,165 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/7c7f87a6124bfe2f4049ee7803e0f98c/B/b4b46e5bd82a454d819133a11c7a80cb, entries=150, sequenceid=79, filesize=11.7 K 2024-11-16T09:58:37,167 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/7c7f87a6124bfe2f4049ee7803e0f98c/.tmp/C/39d84edc31624bc4bca34f4812ae659d as hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/7c7f87a6124bfe2f4049ee7803e0f98c/C/39d84edc31624bc4bca34f4812ae659d 2024-11-16T09:58:37,172 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/7c7f87a6124bfe2f4049ee7803e0f98c/C/39d84edc31624bc4bca34f4812ae659d, entries=150, sequenceid=79, filesize=11.7 K 2024-11-16T09:58:37,174 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~154.31 KB/158010, heapSize ~405 KB/414720, currentSize=53.67 KB/54960 for 7c7f87a6124bfe2f4049ee7803e0f98c in 1419ms, sequenceid=79, compaction requested=true 2024-11-16T09:58:37,174 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 7c7f87a6124bfe2f4049ee7803e0f98c: 2024-11-16T09:58:37,174 DEBUG [RS:0;037c58a19ddb:37267-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-16T09:58:37,175 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 7c7f87a6124bfe2f4049ee7803e0f98c:A, priority=-2147483648, current under compaction store size is 1 2024-11-16T09:58:37,175 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-16T09:58:37,175 DEBUG [RS:0;037c58a19ddb:37267-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-16T09:58:37,176 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 7c7f87a6124bfe2f4049ee7803e0f98c:B, priority=-2147483648, current under compaction store size is 2 2024-11-16T09:58:37,176 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-16T09:58:37,176 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 7c7f87a6124bfe2f4049ee7803e0f98c:C, priority=-2147483648, current under compaction store size is 3 2024-11-16T09:58:37,176 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-16T09:58:37,176 DEBUG [RS:0;037c58a19ddb:37267-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 149602 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-16T09:58:37,176 DEBUG [RS:0;037c58a19ddb:37267-shortCompactions-0 {}] regionserver.HStore(1541): 7c7f87a6124bfe2f4049ee7803e0f98c/A is initiating minor compaction (all files) 2024-11-16T09:58:37,176 INFO [RS:0;037c58a19ddb:37267-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 7c7f87a6124bfe2f4049ee7803e0f98c/A in TestAcidGuarantees,,1731751108989.7c7f87a6124bfe2f4049ee7803e0f98c. 2024-11-16T09:58:37,176 INFO [RS:0;037c58a19ddb:37267-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/7c7f87a6124bfe2f4049ee7803e0f98c/A/c6a8bb2378f148b9ad055005e5046ab4, hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/7c7f87a6124bfe2f4049ee7803e0f98c/A/ee8a20ddfc1549a1b4d7045964ed8bb8, hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/7c7f87a6124bfe2f4049ee7803e0f98c/A/a9b0a7760e764325a134f8bd6fd9c882, hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/7c7f87a6124bfe2f4049ee7803e0f98c/A/8aaa28ea93ee462eaa017eee7d302c00] into tmpdir=hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/7c7f87a6124bfe2f4049ee7803e0f98c/.tmp, totalSize=146.1 K 2024-11-16T09:58:37,177 INFO [RS:0;037c58a19ddb:37267-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=12 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731751108989.7c7f87a6124bfe2f4049ee7803e0f98c. 2024-11-16T09:58:37,177 DEBUG [RS:0;037c58a19ddb:37267-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731751108989.7c7f87a6124bfe2f4049ee7803e0f98c. files: [hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/7c7f87a6124bfe2f4049ee7803e0f98c/A/c6a8bb2378f148b9ad055005e5046ab4, hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/7c7f87a6124bfe2f4049ee7803e0f98c/A/ee8a20ddfc1549a1b4d7045964ed8bb8, hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/7c7f87a6124bfe2f4049ee7803e0f98c/A/a9b0a7760e764325a134f8bd6fd9c882, hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/7c7f87a6124bfe2f4049ee7803e0f98c/A/8aaa28ea93ee462eaa017eee7d302c00] 2024-11-16T09:58:37,178 DEBUG [RS:0;037c58a19ddb:37267-shortCompactions-0 {}] compactions.Compactor(225): Compacting c6a8bb2378f148b9ad055005e5046ab4, keycount=200, bloomtype=ROW, size=38.6 K, encoding=NONE, compression=NONE, seqNum=17, earliestPutTs=1731751112170 2024-11-16T09:58:37,179 DEBUG [RS:0;037c58a19ddb:37267-shortCompactions-0 {}] compactions.Compactor(225): Compacting ee8a20ddfc1549a1b4d7045964ed8bb8, keycount=200, bloomtype=ROW, size=38.6 K, encoding=NONE, compression=NONE, seqNum=41, earliestPutTs=1731751112230 2024-11-16T09:58:37,179 DEBUG [RS:0;037c58a19ddb:37267-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48004 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-16T09:58:37,179 DEBUG [RS:0;037c58a19ddb:37267-longCompactions-0 {}] regionserver.HStore(1541): 7c7f87a6124bfe2f4049ee7803e0f98c/B is initiating minor compaction (all files) 2024-11-16T09:58:37,179 INFO [RS:0;037c58a19ddb:37267-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 7c7f87a6124bfe2f4049ee7803e0f98c/B in TestAcidGuarantees,,1731751108989.7c7f87a6124bfe2f4049ee7803e0f98c. 2024-11-16T09:58:37,179 INFO [RS:0;037c58a19ddb:37267-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/7c7f87a6124bfe2f4049ee7803e0f98c/B/bf8668dd1fd7427caec8099ddc5334f7, hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/7c7f87a6124bfe2f4049ee7803e0f98c/B/5e27822b25b0488bb4324b737dfd6709, hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/7c7f87a6124bfe2f4049ee7803e0f98c/B/1dd7fb59b8be4b04ae256d3db0407388, hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/7c7f87a6124bfe2f4049ee7803e0f98c/B/b4b46e5bd82a454d819133a11c7a80cb] into tmpdir=hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/7c7f87a6124bfe2f4049ee7803e0f98c/.tmp, totalSize=46.9 K 2024-11-16T09:58:37,180 DEBUG [RS:0;037c58a19ddb:37267-shortCompactions-0 {}] compactions.Compactor(225): Compacting a9b0a7760e764325a134f8bd6fd9c882, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=53, earliestPutTs=1731751113405 2024-11-16T09:58:37,180 DEBUG [RS:0;037c58a19ddb:37267-longCompactions-0 {}] compactions.Compactor(225): Compacting bf8668dd1fd7427caec8099ddc5334f7, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=17, earliestPutTs=1731751112178 2024-11-16T09:58:37,180 DEBUG [RS:0;037c58a19ddb:37267-longCompactions-0 {}] compactions.Compactor(225): Compacting 5e27822b25b0488bb4324b737dfd6709, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=41, earliestPutTs=1731751112245 2024-11-16T09:58:37,180 DEBUG [RS:0;037c58a19ddb:37267-shortCompactions-0 {}] compactions.Compactor(225): Compacting 8aaa28ea93ee462eaa017eee7d302c00, keycount=200, bloomtype=ROW, size=38.6 K, encoding=NONE, compression=NONE, seqNum=79, earliestPutTs=1731751114599 2024-11-16T09:58:37,181 DEBUG [RS:0;037c58a19ddb:37267-longCompactions-0 {}] compactions.Compactor(225): Compacting 1dd7fb59b8be4b04ae256d3db0407388, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=53, earliestPutTs=1731751113405 2024-11-16T09:58:37,182 DEBUG [RS:0;037c58a19ddb:37267-longCompactions-0 {}] compactions.Compactor(225): Compacting b4b46e5bd82a454d819133a11c7a80cb, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=79, earliestPutTs=1731751114599 2024-11-16T09:58:37,205 INFO [RS:0;037c58a19ddb:37267-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=7c7f87a6124bfe2f4049ee7803e0f98c] 2024-11-16T09:58:37,208 INFO [RS:0;037c58a19ddb:37267-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 7c7f87a6124bfe2f4049ee7803e0f98c#B#compaction#141 average throughput is 2.18 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-16T09:58:37,208 DEBUG [RS:0;037c58a19ddb:37267-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/7c7f87a6124bfe2f4049ee7803e0f98c/.tmp/B/b471d622e7e145b989bbb16c6e434c3c is 50, key is test_row_0/B:col10/1731751114601/Put/seqid=0 2024-11-16T09:58:37,221 DEBUG [RS:0;037c58a19ddb:37267-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241116e6fe4c1af66741a783e1743a5e3b07c2_7c7f87a6124bfe2f4049ee7803e0f98c store=[table=TestAcidGuarantees family=A region=7c7f87a6124bfe2f4049ee7803e0f98c] 2024-11-16T09:58:37,227 DEBUG [RS:0;037c58a19ddb:37267-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241116e6fe4c1af66741a783e1743a5e3b07c2_7c7f87a6124bfe2f4049ee7803e0f98c, store=[table=TestAcidGuarantees family=A region=7c7f87a6124bfe2f4049ee7803e0f98c] 2024-11-16T09:58:37,227 DEBUG [RS:0;037c58a19ddb:37267-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241116e6fe4c1af66741a783e1743a5e3b07c2_7c7f87a6124bfe2f4049ee7803e0f98c because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=7c7f87a6124bfe2f4049ee7803e0f98c] 2024-11-16T09:58:37,255 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37591 is added to blk_1073741986_1162 (size=4469) 2024-11-16T09:58:37,257 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=37267 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=41 2024-11-16T09:58:37,257 DEBUG [RS_FLUSH_OPERATIONS-regionserver/037c58a19ddb:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731751108989.7c7f87a6124bfe2f4049ee7803e0f98c. 2024-11-16T09:58:37,257 INFO [RS_FLUSH_OPERATIONS-regionserver/037c58a19ddb:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HRegion(2902): Flushing 7c7f87a6124bfe2f4049ee7803e0f98c 3/3 column families, dataSize=53.67 KB heapSize=141.38 KB 2024-11-16T09:58:37,278 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37591 is added to blk_1073741985_1161 (size=12139) 2024-11-16T09:58:37,306 DEBUG [RS_FLUSH_OPERATIONS-regionserver/037c58a19ddb:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241116e652a28518f24ba7b89f699d5f6a7615_7c7f87a6124bfe2f4049ee7803e0f98c is 50, key is test_row_0/A:col10/1731751115756/Put/seqid=0 2024-11-16T09:58:37,340 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37591 is added to blk_1073741987_1163 (size=12154) 2024-11-16T09:58:37,341 DEBUG [RS_FLUSH_OPERATIONS-regionserver/037c58a19ddb:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T09:58:37,351 INFO [RS_FLUSH_OPERATIONS-regionserver/037c58a19ddb:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241116e652a28518f24ba7b89f699d5f6a7615_7c7f87a6124bfe2f4049ee7803e0f98c to hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241116e652a28518f24ba7b89f699d5f6a7615_7c7f87a6124bfe2f4049ee7803e0f98c 2024-11-16T09:58:37,355 DEBUG [RS_FLUSH_OPERATIONS-regionserver/037c58a19ddb:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/7c7f87a6124bfe2f4049ee7803e0f98c/.tmp/A/d4eba6854cf24b93ab8094b32b349735, store: [table=TestAcidGuarantees family=A region=7c7f87a6124bfe2f4049ee7803e0f98c] 2024-11-16T09:58:37,355 DEBUG [RS_FLUSH_OPERATIONS-regionserver/037c58a19ddb:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/7c7f87a6124bfe2f4049ee7803e0f98c/.tmp/A/d4eba6854cf24b93ab8094b32b349735 is 175, key is test_row_0/A:col10/1731751115756/Put/seqid=0 2024-11-16T09:58:37,367 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37591 is added to blk_1073741988_1164 (size=30955) 2024-11-16T09:58:37,368 INFO [RS_FLUSH_OPERATIONS-regionserver/037c58a19ddb:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=90, memsize=17.9 K, hasBloomFilter=true, into tmp file hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/7c7f87a6124bfe2f4049ee7803e0f98c/.tmp/A/d4eba6854cf24b93ab8094b32b349735 2024-11-16T09:58:37,382 DEBUG [RS_FLUSH_OPERATIONS-regionserver/037c58a19ddb:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/7c7f87a6124bfe2f4049ee7803e0f98c/.tmp/B/85b3c54b164145d2925f3a8bb12ce8fe is 50, key is test_row_0/B:col10/1731751115756/Put/seqid=0 2024-11-16T09:58:37,421 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37591 is added to blk_1073741989_1165 (size=12001) 2024-11-16T09:58:37,422 INFO [RS_FLUSH_OPERATIONS-regionserver/037c58a19ddb:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=90 (bloomFilter=true), to=hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/7c7f87a6124bfe2f4049ee7803e0f98c/.tmp/B/85b3c54b164145d2925f3a8bb12ce8fe 2024-11-16T09:58:37,433 DEBUG [RS_FLUSH_OPERATIONS-regionserver/037c58a19ddb:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/7c7f87a6124bfe2f4049ee7803e0f98c/.tmp/C/0b8fa5e40ba242e6a84abbb9e8e4fea5 is 50, key is test_row_0/C:col10/1731751115756/Put/seqid=0 2024-11-16T09:58:37,457 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37591 is added to blk_1073741990_1166 (size=12001) 2024-11-16T09:58:37,458 INFO [RS_FLUSH_OPERATIONS-regionserver/037c58a19ddb:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=90 (bloomFilter=true), to=hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/7c7f87a6124bfe2f4049ee7803e0f98c/.tmp/C/0b8fa5e40ba242e6a84abbb9e8e4fea5 2024-11-16T09:58:37,463 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37877 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=40 2024-11-16T09:58:37,468 DEBUG [RS_FLUSH_OPERATIONS-regionserver/037c58a19ddb:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/7c7f87a6124bfe2f4049ee7803e0f98c/.tmp/A/d4eba6854cf24b93ab8094b32b349735 as hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/7c7f87a6124bfe2f4049ee7803e0f98c/A/d4eba6854cf24b93ab8094b32b349735 2024-11-16T09:58:37,489 INFO [RS_FLUSH_OPERATIONS-regionserver/037c58a19ddb:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/7c7f87a6124bfe2f4049ee7803e0f98c/A/d4eba6854cf24b93ab8094b32b349735, entries=150, sequenceid=90, filesize=30.2 K 2024-11-16T09:58:37,497 DEBUG [RS_FLUSH_OPERATIONS-regionserver/037c58a19ddb:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/7c7f87a6124bfe2f4049ee7803e0f98c/.tmp/B/85b3c54b164145d2925f3a8bb12ce8fe as hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/7c7f87a6124bfe2f4049ee7803e0f98c/B/85b3c54b164145d2925f3a8bb12ce8fe 2024-11-16T09:58:37,499 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T09:58:37,500 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T09:58:37,501 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T09:58:37,503 INFO [RS_FLUSH_OPERATIONS-regionserver/037c58a19ddb:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/7c7f87a6124bfe2f4049ee7803e0f98c/B/85b3c54b164145d2925f3a8bb12ce8fe, entries=150, sequenceid=90, filesize=11.7 K 2024-11-16T09:58:37,504 DEBUG [RS_FLUSH_OPERATIONS-regionserver/037c58a19ddb:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/7c7f87a6124bfe2f4049ee7803e0f98c/.tmp/C/0b8fa5e40ba242e6a84abbb9e8e4fea5 as hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/7c7f87a6124bfe2f4049ee7803e0f98c/C/0b8fa5e40ba242e6a84abbb9e8e4fea5 2024-11-16T09:58:37,508 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T09:58:37,508 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T09:58:37,510 INFO [RS_FLUSH_OPERATIONS-regionserver/037c58a19ddb:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/7c7f87a6124bfe2f4049ee7803e0f98c/C/0b8fa5e40ba242e6a84abbb9e8e4fea5, entries=150, sequenceid=90, filesize=11.7 K 2024-11-16T09:58:37,510 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T09:58:37,511 INFO [RS_FLUSH_OPERATIONS-regionserver/037c58a19ddb:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~141.33 KB/144720, currentSize=0 B/0 for 7c7f87a6124bfe2f4049ee7803e0f98c in 254ms, sequenceid=90, compaction requested=true 2024-11-16T09:58:37,511 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T09:58:37,511 DEBUG [RS_FLUSH_OPERATIONS-regionserver/037c58a19ddb:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HRegion(2603): Flush status journal for 7c7f87a6124bfe2f4049ee7803e0f98c: 2024-11-16T09:58:37,511 DEBUG [RS_FLUSH_OPERATIONS-regionserver/037c58a19ddb:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731751108989.7c7f87a6124bfe2f4049ee7803e0f98c. 2024-11-16T09:58:37,511 DEBUG [RS_FLUSH_OPERATIONS-regionserver/037c58a19ddb:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=41 2024-11-16T09:58:37,511 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T09:58:37,512 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37877 {}] master.HMaster(4169): Remote procedure done, pid=41 2024-11-16T09:58:37,512 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T09:58:37,512 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T09:58:37,513 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T09:58:37,513 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T09:58:37,514 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T09:58:37,514 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T09:58:37,515 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T09:58:37,516 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=41, resume processing ppid=40 2024-11-16T09:58:37,516 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=41, ppid=40, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.1850 sec 2024-11-16T09:58:37,516 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T09:58:37,516 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T09:58:37,517 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T09:58:37,518 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T09:58:37,518 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T09:58:37,519 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T09:58:37,519 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T09:58:37,519 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T09:58:37,520 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T09:58:37,520 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T09:58:37,521 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T09:58:37,521 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T09:58:37,522 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T09:58:37,522 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T09:58:37,523 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T09:58:37,523 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=40, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=40, table=TestAcidGuarantees in 1.1930 sec 2024-11-16T09:58:37,523 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T09:58:37,524 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T09:58:37,524 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T09:58:37,525 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T09:58:37,528 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T09:58:37,529 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T09:58:37,529 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T09:58:37,530 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T09:58:37,530 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T09:58:37,531 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T09:58:37,532 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T09:58:37,532 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T09:58:37,533 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T09:58:37,533 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T09:58:37,534 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T09:58:37,534 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T09:58:37,534 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T09:58:37,535 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T09:58:37,535 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T09:58:37,536 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T09:58:37,536 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T09:58:37,536 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T09:58:37,537 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T09:58:37,537 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T09:58:37,538 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T09:58:37,538 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T09:58:37,539 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T09:58:37,539 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T09:58:37,539 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T09:58:37,540 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T09:58:37,541 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T09:58:37,542 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T09:58:37,543 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T09:58:37,544 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T09:58:37,544 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T09:58:37,545 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T09:58:37,546 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T09:58:37,546 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T09:58:37,547 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T09:58:37,547 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T09:58:37,547 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T09:58:37,548 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T09:58:37,548 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T09:58:37,548 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T09:58:37,549 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T09:58:37,550 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T09:58:37,550 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T09:58:37,551 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T09:58:37,551 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T09:58:37,552 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T09:58:37,552 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T09:58:37,553 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T09:58:37,553 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T09:58:37,554 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T09:58:37,554 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T09:58:37,554 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T09:58:37,555 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T09:58:37,555 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T09:58:37,556 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T09:58:37,556 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T09:58:37,557 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T09:58:37,557 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T09:58:37,557 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T09:58:37,558 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T09:58:37,558 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T09:58:37,558 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T09:58:37,558 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T09:58:37,559 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T09:58:37,559 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T09:58:37,560 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T09:58:37,560 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T09:58:37,562 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T09:58:37,562 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T09:58:37,562 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T09:58:37,563 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T09:58:37,564 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T09:58:37,564 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T09:58:37,564 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T09:58:37,564 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T09:58:37,565 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T09:58:37,565 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T09:58:37,565 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T09:58:37,566 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T09:58:37,566 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T09:58:37,567 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T09:58:37,567 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T09:58:37,567 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T09:58:37,568 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T09:58:37,568 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T09:58:37,569 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T09:58:37,569 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T09:58:37,570 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T09:58:37,570 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T09:58:37,570 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T09:58:37,571 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T09:58:37,571 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T09:58:37,572 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T09:58:37,573 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T09:58:37,573 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T09:58:37,574 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T09:58:37,574 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T09:58:37,574 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T09:58:37,575 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T09:58:37,575 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T09:58:37,576 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T09:58:37,577 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T09:58:37,577 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T09:58:37,578 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T09:58:37,578 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T09:58:37,579 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T09:58:37,579 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T09:58:37,580 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T09:58:37,580 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T09:58:37,581 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T09:58:37,582 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T09:58:37,583 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T09:58:37,583 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T09:58:37,584 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T09:58:37,584 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T09:58:37,584 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T09:58:37,585 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T09:58:37,585 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T09:58:37,586 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T09:58:37,586 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T09:58:37,586 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T09:58:37,587 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T09:58:37,587 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T09:58:37,588 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T09:58:37,588 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T09:58:37,588 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T09:58:37,589 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T09:58:37,589 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T09:58:37,590 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T09:58:37,591 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T09:58:37,591 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T09:58:37,592 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T09:58:37,592 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T09:58:37,593 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T09:58:37,594 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T09:58:37,594 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T09:58:37,594 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T09:58:37,595 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T09:58:37,596 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T09:58:37,596 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T09:58:37,596 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T09:58:37,597 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T09:58:37,597 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T09:58:37,597 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T09:58:37,597 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T09:58:37,598 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T09:58:37,598 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T09:58:37,599 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T09:58:37,599 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T09:58:37,599 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T09:58:37,600 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T09:58:37,600 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T09:58:37,600 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T09:58:37,601 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T09:58:37,601 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T09:58:37,602 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T09:58:37,602 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T09:58:37,602 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T09:58:37,603 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T09:58:37,603 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T09:58:37,603 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T09:58:37,603 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T09:58:37,604 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T09:58:37,604 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T09:58:37,605 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T09:58:37,605 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T09:58:37,605 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T09:58:37,606 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T09:58:37,606 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T09:58:37,606 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T09:58:37,607 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T09:58:37,607 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T09:58:37,607 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T09:58:37,608 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T09:58:37,608 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T09:58:37,608 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T09:58:37,609 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T09:58:37,609 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T09:58:37,610 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T09:58:37,610 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T09:58:37,610 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T09:58:37,611 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T09:58:37,611 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T09:58:37,612 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T09:58:37,612 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T09:58:37,613 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T09:58:37,613 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T09:58:37,614 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T09:58:37,614 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T09:58:37,615 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T09:58:37,615 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T09:58:37,615 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T09:58:37,615 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T09:58:37,616 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T09:58:37,616 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T09:58:37,617 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T09:58:37,617 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T09:58:37,617 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T09:58:37,618 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T09:58:37,618 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T09:58:37,618 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T09:58:37,619 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T09:58:37,619 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T09:58:37,619 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T09:58:37,620 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T09:58:37,620 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T09:58:37,621 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T09:58:37,621 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T09:58:37,621 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T09:58:37,621 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T09:58:37,622 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T09:58:37,622 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T09:58:37,623 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T09:58:37,623 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T09:58:37,624 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T09:58:37,624 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T09:58:37,624 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T09:58:37,624 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T09:58:37,625 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T09:58:37,625 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T09:58:37,625 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T09:58:37,626 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T09:58:37,626 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T09:58:37,626 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T09:58:37,627 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T09:58:37,627 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T09:58:37,627 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T09:58:37,628 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T09:58:37,628 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T09:58:37,628 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T09:58:37,629 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T09:58:37,629 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T09:58:37,629 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T09:58:37,630 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T09:58:37,630 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T09:58:37,630 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T09:58:37,631 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T09:58:37,631 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T09:58:37,631 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T09:58:37,631 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T09:58:37,631 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T09:58:37,632 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T09:58:37,632 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T09:58:37,633 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T09:58:37,634 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T09:58:37,634 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T09:58:37,635 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T09:58:37,635 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T09:58:37,635 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T09:58:37,635 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T09:58:37,636 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T09:58:37,636 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T09:58:37,636 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T09:58:37,637 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T09:58:37,637 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T09:58:37,637 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T09:58:37,637 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T09:58:37,638 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T09:58:37,638 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T09:58:37,638 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T09:58:37,639 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T09:58:37,639 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T09:58:37,639 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T09:58:37,640 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T09:58:37,641 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T09:58:37,641 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T09:58:37,641 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T09:58:37,642 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T09:58:37,642 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T09:58:37,642 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T09:58:37,643 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T09:58:37,643 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T09:58:37,643 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T09:58:37,643 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T09:58:37,644 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T09:58:37,645 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T09:58:37,646 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T09:58:37,646 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T09:58:37,647 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T09:58:37,647 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T09:58:37,647 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T09:58:37,647 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T09:58:37,647 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T09:58:37,648 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T09:58:37,649 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T09:58:37,649 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T09:58:37,650 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T09:58:37,650 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T09:58:37,651 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T09:58:37,651 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T09:58:37,652 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T09:58:37,652 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T09:58:37,653 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T09:58:37,654 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T09:58:37,655 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T09:58:37,656 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T09:58:37,656 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T09:58:37,656 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T09:58:37,657 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T09:58:37,657 INFO [RS:0;037c58a19ddb:37267-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 7c7f87a6124bfe2f4049ee7803e0f98c#A#compaction#142 average throughput is 0.05 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-16T09:58:37,657 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T09:58:37,658 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T09:58:37,658 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T09:58:37,658 DEBUG [RS:0;037c58a19ddb:37267-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/7c7f87a6124bfe2f4049ee7803e0f98c/.tmp/A/decd818193a64048853aa6c79bdf0976 is 175, key is test_row_0/A:col10/1731751114601/Put/seqid=0 2024-11-16T09:58:37,659 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T09:58:37,659 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T09:58:37,659 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T09:58:37,659 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T09:58:37,660 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T09:58:37,660 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T09:58:37,660 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T09:58:37,661 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T09:58:37,661 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T09:58:37,661 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T09:58:37,662 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T09:58:37,662 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T09:58:37,663 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T09:58:37,663 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T09:58:37,664 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T09:58:37,664 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T09:58:37,665 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T09:58:37,665 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T09:58:37,666 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T09:58:37,666 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T09:58:37,666 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T09:58:37,667 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T09:58:37,668 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T09:58:37,668 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T09:58:37,668 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T09:58:37,669 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T09:58:37,670 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T09:58:37,670 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T09:58:37,671 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T09:58:37,678 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T09:58:37,680 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T09:58:37,682 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T09:58:37,683 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T09:58:37,683 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T09:58:37,684 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T09:58:37,684 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T09:58:37,685 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T09:58:37,686 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T09:58:37,687 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T09:58:37,687 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T09:58:37,688 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T09:58:37,689 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T09:58:37,693 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T09:58:37,694 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T09:58:37,695 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T09:58:37,696 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T09:58:37,697 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37591 is added to blk_1073741991_1167 (size=31093) 2024-11-16T09:58:37,700 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T09:58:37,701 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T09:58:37,702 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T09:58:37,703 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T09:58:37,704 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T09:58:37,705 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T09:58:37,706 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T09:58:37,708 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T09:58:37,708 DEBUG [RS:0;037c58a19ddb:37267-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/7c7f87a6124bfe2f4049ee7803e0f98c/.tmp/A/decd818193a64048853aa6c79bdf0976 as hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/7c7f87a6124bfe2f4049ee7803e0f98c/A/decd818193a64048853aa6c79bdf0976 2024-11-16T09:58:37,709 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T09:58:37,710 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T09:58:37,711 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T09:58:37,711 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T09:58:37,712 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T09:58:37,713 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T09:58:37,714 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T09:58:37,715 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T09:58:37,715 INFO [RS:0;037c58a19ddb:37267-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 7c7f87a6124bfe2f4049ee7803e0f98c/A of 7c7f87a6124bfe2f4049ee7803e0f98c into decd818193a64048853aa6c79bdf0976(size=30.4 K), total size for store is 60.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-16T09:58:37,715 DEBUG [RS:0;037c58a19ddb:37267-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 7c7f87a6124bfe2f4049ee7803e0f98c: 2024-11-16T09:58:37,715 INFO [RS:0;037c58a19ddb:37267-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731751108989.7c7f87a6124bfe2f4049ee7803e0f98c., storeName=7c7f87a6124bfe2f4049ee7803e0f98c/A, priority=12, startTime=1731751117174; duration=0sec 2024-11-16T09:58:37,715 DEBUG [RS:0;037c58a19ddb:37267-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-16T09:58:37,715 DEBUG [RS:0;037c58a19ddb:37267-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 7c7f87a6124bfe2f4049ee7803e0f98c:A 2024-11-16T09:58:37,715 DEBUG [RS:0;037c58a19ddb:37267-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 5 store files, 0 compacting, 5 eligible, 16 blocking 2024-11-16T09:58:37,716 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T09:58:37,717 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T09:58:37,718 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T09:58:37,718 DEBUG [RS:0;037c58a19ddb:37267-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 5 files of size 60005 starting at candidate #0 after considering 6 permutations with 6 in ratio 2024-11-16T09:58:37,718 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T09:58:37,718 DEBUG [RS:0;037c58a19ddb:37267-shortCompactions-0 {}] regionserver.HStore(1541): 7c7f87a6124bfe2f4049ee7803e0f98c/C is initiating minor compaction (all files) 2024-11-16T09:58:37,719 INFO [RS:0;037c58a19ddb:37267-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 7c7f87a6124bfe2f4049ee7803e0f98c/C in TestAcidGuarantees,,1731751108989.7c7f87a6124bfe2f4049ee7803e0f98c. 2024-11-16T09:58:37,719 INFO [RS:0;037c58a19ddb:37267-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/7c7f87a6124bfe2f4049ee7803e0f98c/C/ccfea16deb0641a3ab89085b418edb9b, hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/7c7f87a6124bfe2f4049ee7803e0f98c/C/95998a9414674875803abd21d398ade9, hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/7c7f87a6124bfe2f4049ee7803e0f98c/C/8ee49e7518574d3b98cb848b8d210202, hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/7c7f87a6124bfe2f4049ee7803e0f98c/C/39d84edc31624bc4bca34f4812ae659d, hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/7c7f87a6124bfe2f4049ee7803e0f98c/C/0b8fa5e40ba242e6a84abbb9e8e4fea5] into tmpdir=hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/7c7f87a6124bfe2f4049ee7803e0f98c/.tmp, totalSize=58.6 K 2024-11-16T09:58:37,719 DEBUG [RS:0;037c58a19ddb:37267-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/7c7f87a6124bfe2f4049ee7803e0f98c/.tmp/B/b471d622e7e145b989bbb16c6e434c3c as hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/7c7f87a6124bfe2f4049ee7803e0f98c/B/b471d622e7e145b989bbb16c6e434c3c 2024-11-16T09:58:37,719 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T09:58:37,720 DEBUG [RS:0;037c58a19ddb:37267-shortCompactions-0 {}] compactions.Compactor(225): Compacting ccfea16deb0641a3ab89085b418edb9b, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=17, earliestPutTs=1731751112178 2024-11-16T09:58:37,720 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T09:58:37,720 DEBUG [RS:0;037c58a19ddb:37267-shortCompactions-0 {}] compactions.Compactor(225): Compacting 95998a9414674875803abd21d398ade9, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=41, earliestPutTs=1731751112245 2024-11-16T09:58:37,720 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T09:58:37,721 DEBUG [RS:0;037c58a19ddb:37267-shortCompactions-0 {}] compactions.Compactor(225): Compacting 8ee49e7518574d3b98cb848b8d210202, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=53, earliestPutTs=1731751113405 2024-11-16T09:58:37,721 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T09:58:37,722 DEBUG [RS:0;037c58a19ddb:37267-shortCompactions-0 {}] compactions.Compactor(225): Compacting 39d84edc31624bc4bca34f4812ae659d, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=79, earliestPutTs=1731751114599 2024-11-16T09:58:37,722 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T09:58:37,722 DEBUG [RS:0;037c58a19ddb:37267-shortCompactions-0 {}] compactions.Compactor(225): Compacting 0b8fa5e40ba242e6a84abbb9e8e4fea5, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=90, earliestPutTs=1731751115756 2024-11-16T09:58:37,722 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T09:58:37,723 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T09:58:37,723 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T09:58:37,724 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T09:58:37,724 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T09:58:37,725 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T09:58:37,725 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T09:58:37,725 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T09:58:37,726 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T09:58:37,726 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T09:58:37,727 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T09:58:37,727 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T09:58:37,728 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T09:58:37,728 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T09:58:37,728 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T09:58:37,729 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T09:58:37,729 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T09:58:37,729 INFO [RS:0;037c58a19ddb:37267-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 7c7f87a6124bfe2f4049ee7803e0f98c/B of 7c7f87a6124bfe2f4049ee7803e0f98c into b471d622e7e145b989bbb16c6e434c3c(size=11.9 K), total size for store is 23.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-16T09:58:37,729 DEBUG [RS:0;037c58a19ddb:37267-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 7c7f87a6124bfe2f4049ee7803e0f98c: 2024-11-16T09:58:37,730 INFO [RS:0;037c58a19ddb:37267-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731751108989.7c7f87a6124bfe2f4049ee7803e0f98c., storeName=7c7f87a6124bfe2f4049ee7803e0f98c/B, priority=12, startTime=1731751117175; duration=0sec 2024-11-16T09:58:37,730 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T09:58:37,730 DEBUG [RS:0;037c58a19ddb:37267-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-16T09:58:37,730 DEBUG [RS:0;037c58a19ddb:37267-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 7c7f87a6124bfe2f4049ee7803e0f98c:B 2024-11-16T09:58:37,730 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T09:58:37,730 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T09:58:37,731 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T09:58:37,731 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T09:58:37,731 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T09:58:37,732 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T09:58:37,732 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T09:58:37,733 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T09:58:37,733 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T09:58:37,733 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T09:58:37,734 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T09:58:37,734 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T09:58:37,735 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T09:58:37,735 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T09:58:37,735 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T09:58:37,735 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T09:58:37,736 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T09:58:37,737 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T09:58:37,737 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T09:58:37,738 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T09:58:37,738 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T09:58:37,738 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T09:58:37,739 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T09:58:37,739 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T09:58:37,739 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T09:58:37,739 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T09:58:37,740 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T09:58:37,740 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T09:58:37,740 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T09:58:37,741 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T09:58:37,741 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T09:58:37,742 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T09:58:37,742 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T09:58:37,743 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T09:58:37,743 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T09:58:37,744 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T09:58:37,744 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T09:58:37,745 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T09:58:37,745 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T09:58:37,746 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T09:58:37,747 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T09:58:37,747 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T09:58:37,748 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T09:58:37,748 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T09:58:37,749 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T09:58:37,749 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T09:58:37,750 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T09:58:37,750 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T09:58:37,751 INFO [RS:0;037c58a19ddb:37267-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 7c7f87a6124bfe2f4049ee7803e0f98c#C#compaction#146 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-16T09:58:37,751 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T09:58:37,752 DEBUG [RS:0;037c58a19ddb:37267-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/7c7f87a6124bfe2f4049ee7803e0f98c/.tmp/C/f7adaf3740b84fe18c83f162e49403a7 is 50, key is test_row_0/C:col10/1731751115756/Put/seqid=0 2024-11-16T09:58:37,752 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T09:58:37,753 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T09:58:37,753 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T09:58:37,754 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T09:58:37,754 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T09:58:37,754 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T09:58:37,755 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T09:58:37,756 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T09:58:37,756 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T09:58:37,757 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T09:58:37,758 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T09:58:37,759 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T09:58:37,760 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T09:58:37,760 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T09:58:37,761 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T09:58:37,761 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T09:58:37,761 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T09:58:37,762 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T09:58:37,762 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T09:58:37,762 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T09:58:37,763 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T09:58:37,763 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T09:58:37,764 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T09:58:37,764 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T09:58:37,764 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T09:58:37,765 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T09:58:37,765 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T09:58:37,765 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T09:58:37,766 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T09:58:37,766 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T09:58:37,767 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T09:58:37,767 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T09:58:37,767 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T09:58:37,768 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T09:58:37,768 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T09:58:37,769 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T09:58:37,770 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T09:58:37,771 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T09:58:37,771 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T09:58:37,772 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T09:58:37,772 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T09:58:37,773 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T09:58:37,774 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T09:58:37,777 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T09:58:37,779 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T09:58:37,779 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T09:58:37,779 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T09:58:37,780 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T09:58:37,781 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T09:58:37,781 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T09:58:37,781 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T09:58:37,782 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T09:58:37,782 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T09:58:37,783 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T09:58:37,783 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T09:58:37,783 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T09:58:37,783 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T09:58:37,783 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T09:58:37,784 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T09:58:37,784 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T09:58:37,785 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T09:58:37,785 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T09:58:37,785 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T09:58:37,786 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T09:58:37,786 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T09:58:37,787 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T09:58:37,787 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T09:58:37,787 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T09:58:37,788 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T09:58:37,789 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T09:58:37,789 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T09:58:37,789 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37591 is added to blk_1073741992_1168 (size=12173) 2024-11-16T09:58:37,790 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T09:58:37,790 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T09:58:37,791 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T09:58:37,791 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T09:58:37,792 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T09:58:37,792 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T09:58:37,793 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T09:58:37,793 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T09:58:37,793 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T09:58:37,794 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T09:58:37,794 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T09:58:37,795 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T09:58:37,795 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T09:58:37,796 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T09:58:37,796 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T09:58:37,797 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T09:58:37,797 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T09:58:37,798 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T09:58:37,798 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T09:58:37,799 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T09:58:37,799 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T09:58:37,800 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T09:58:37,800 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T09:58:37,800 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T09:58:37,801 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T09:58:37,801 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T09:58:37,802 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T09:58:37,802 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T09:58:37,803 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T09:58:37,803 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T09:58:37,803 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T09:58:37,804 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T09:58:37,804 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T09:58:37,805 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T09:58:37,805 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T09:58:37,806 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T09:58:37,806 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T09:58:37,806 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T09:58:37,807 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T09:58:37,807 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T09:58:37,808 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T09:58:37,809 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T09:58:37,809 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T09:58:37,810 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T09:58:37,810 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T09:58:37,810 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T09:58:37,811 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T09:58:37,812 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T09:58:37,812 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T09:58:37,812 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T09:58:37,814 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T09:58:37,814 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T09:58:37,814 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T09:58:37,815 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T09:58:37,816 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T09:58:37,816 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T09:58:37,817 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T09:58:37,817 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T09:58:37,818 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T09:58:37,819 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T09:58:37,819 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T09:58:37,820 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T09:58:37,820 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T09:58:37,821 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T09:58:37,821 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T09:58:37,822 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T09:58:37,822 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T09:58:37,823 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T09:58:37,824 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T09:58:37,824 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T09:58:37,825 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T09:58:37,825 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T09:58:37,826 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T09:58:37,826 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T09:58:37,827 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T09:58:37,827 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T09:58:37,828 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T09:58:37,828 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T09:58:37,829 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T09:58:37,830 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T09:58:37,830 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T09:58:37,831 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T09:58:37,831 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T09:58:37,832 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T09:58:37,832 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T09:58:37,832 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T09:58:37,834 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T09:58:37,834 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T09:58:37,834 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T09:58:37,834 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T09:58:37,835 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T09:58:37,835 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T09:58:37,836 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T09:58:37,836 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T09:58:37,837 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T09:58:37,838 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T09:58:37,838 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T09:58:37,839 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T09:58:37,839 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T09:58:37,841 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T09:58:37,841 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T09:58:37,842 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T09:58:37,843 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T09:58:37,843 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T09:58:37,844 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T09:58:37,844 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T09:58:37,845 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T09:58:37,845 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T09:58:37,846 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T09:58:37,846 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T09:58:37,847 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T09:58:37,847 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T09:58:37,847 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T09:58:37,848 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T09:58:37,849 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T09:58:37,849 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T09:58:37,849 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T09:58:37,850 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T09:58:37,850 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T09:58:37,851 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T09:58:37,851 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T09:58:37,852 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T09:58:37,853 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T09:58:37,853 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T09:58:37,853 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T09:58:37,854 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T09:58:37,854 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T09:58:37,855 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T09:58:37,855 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T09:58:37,856 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T09:58:37,857 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T09:58:37,857 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T09:58:37,857 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T09:58:37,858 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T09:58:37,859 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T09:58:37,859 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T09:58:37,859 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T09:58:37,860 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T09:58:37,860 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T09:58:37,861 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T09:58:37,861 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T09:58:37,862 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T09:58:37,862 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T09:58:37,863 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T09:58:37,863 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T09:58:37,864 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T09:58:37,864 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T09:58:37,864 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T09:58:37,865 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T09:58:37,865 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T09:58:37,866 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T09:58:37,866 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T09:58:37,867 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T09:58:37,867 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T09:58:37,868 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T09:58:37,869 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T09:58:37,869 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T09:58:37,870 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T09:58:37,870 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T09:58:37,871 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T09:58:37,871 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T09:58:37,871 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T09:58:37,872 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T09:58:37,873 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T09:58:37,874 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T09:58:37,874 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T09:58:37,875 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T09:58:37,875 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T09:58:37,876 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T09:58:37,876 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T09:58:37,877 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T09:58:37,877 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T09:58:37,878 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T09:58:37,879 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T09:58:37,879 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T09:58:37,880 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T09:58:37,880 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T09:58:37,880 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T09:58:37,881 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T09:58:37,882 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T09:58:37,882 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T09:58:37,883 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T09:58:37,883 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T09:58:37,884 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T09:58:37,884 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T09:58:37,884 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T09:58:37,885 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T09:58:37,886 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T09:58:37,887 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T09:58:37,888 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T09:58:37,888 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T09:58:37,888 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T09:58:37,889 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T09:58:37,890 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T09:58:37,890 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T09:58:37,891 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T09:58:37,891 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T09:58:37,891 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T09:58:37,892 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T09:58:37,892 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T09:58:37,893 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T09:58:37,894 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T09:58:37,894 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T09:58:37,894 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T09:58:37,895 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T09:58:37,896 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T09:58:37,896 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T09:58:37,897 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T09:58:37,897 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T09:58:37,898 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T09:58:37,898 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T09:58:37,898 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T09:58:37,899 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T09:58:37,899 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T09:58:37,900 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T09:58:37,900 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T09:58:37,901 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T09:58:37,902 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T09:58:37,902 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T09:58:37,903 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T09:58:37,903 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T09:58:37,903 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T09:58:37,904 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T09:58:37,905 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T09:58:37,905 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T09:58:37,906 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T09:58:37,906 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T09:58:37,907 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T09:58:37,907 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T09:58:37,907 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T09:58:37,908 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T09:58:37,909 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T09:58:37,909 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T09:58:37,909 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T09:58:37,910 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T09:58:37,910 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T09:58:37,911 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T09:58:37,912 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T09:58:37,912 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T09:58:37,912 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T09:58:37,913 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T09:58:37,913 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T09:58:37,914 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T09:58:37,914 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T09:58:37,915 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T09:58:37,915 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T09:58:37,915 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T09:58:37,917 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T09:58:37,917 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T09:58:37,918 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T09:58:37,918 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T09:58:37,919 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T09:58:37,919 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T09:58:37,920 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T09:58:37,920 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T09:58:37,921 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T09:58:37,921 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T09:58:37,922 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T09:58:37,922 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T09:58:37,923 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T09:58:37,924 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T09:58:37,924 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T09:58:37,925 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T09:58:37,926 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T09:58:37,926 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T09:58:37,926 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T09:58:37,927 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T09:58:37,927 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T09:58:37,928 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T09:58:37,929 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T09:58:37,929 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T09:58:37,930 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T09:58:37,930 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T09:58:37,931 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T09:58:37,931 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T09:58:37,932 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T09:58:37,932 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T09:58:37,933 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T09:58:37,933 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T09:58:37,933 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T09:58:37,934 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T09:58:37,935 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T09:58:37,935 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T09:58:37,936 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T09:58:37,936 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T09:58:37,937 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T09:58:37,937 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T09:58:37,938 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T09:58:37,938 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T09:58:37,939 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T09:58:37,939 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T09:58:37,939 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T09:58:37,940 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T09:58:37,940 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T09:58:37,941 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T09:58:37,942 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T09:58:37,942 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T09:58:37,943 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T09:58:37,944 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T09:58:37,944 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T09:58:37,945 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T09:58:37,945 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T09:58:37,946 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T09:58:37,947 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T09:58:37,947 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T09:58:37,946 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T09:58:37,948 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T09:58:37,949 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T09:58:37,949 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T09:58:37,950 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T09:58:37,950 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T09:58:37,950 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T09:58:37,951 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T09:58:37,951 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T09:58:37,952 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T09:58:37,952 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T09:58:37,953 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T09:58:37,953 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T09:58:37,954 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T09:58:37,954 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T09:58:37,955 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T09:58:37,956 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T09:58:37,957 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T09:58:37,957 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T09:58:37,958 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T09:58:37,958 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T09:58:37,959 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T09:58:37,959 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T09:58:37,960 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T09:58:37,960 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T09:58:37,961 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T09:58:37,961 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T09:58:37,961 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T09:58:37,961 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T09:58:37,962 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T09:58:37,962 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T09:58:37,963 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T09:58:37,963 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T09:58:37,963 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T09:58:37,964 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T09:58:37,964 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T09:58:37,964 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T09:58:37,964 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T09:58:37,965 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T09:58:37,966 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T09:58:37,967 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T09:58:37,968 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T09:58:37,968 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T09:58:37,969 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T09:58:37,970 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T09:58:37,970 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T09:58:37,970 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T09:58:37,971 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T09:58:37,971 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T09:58:37,971 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T09:58:37,972 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T09:58:37,972 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T09:58:37,973 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T09:58:37,973 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T09:58:37,973 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T09:58:37,974 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T09:58:37,974 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T09:58:37,974 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T09:58:37,975 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T09:58:37,975 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T09:58:37,975 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T09:58:37,976 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T09:58:37,976 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T09:58:37,976 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T09:58:37,977 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T09:58:37,977 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T09:58:37,977 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T09:58:37,978 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T09:58:37,978 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T09:58:37,978 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T09:58:37,979 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T09:58:37,979 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T09:58:37,979 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T09:58:37,979 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T09:58:37,980 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T09:58:37,981 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T09:58:37,981 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T09:58:37,981 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T09:58:37,982 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T09:58:37,982 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T09:58:37,982 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T09:58:37,983 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T09:58:37,983 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T09:58:37,983 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T09:58:37,984 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T09:58:37,984 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T09:58:37,985 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T09:58:37,985 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T09:58:37,985 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T09:58:37,986 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T09:58:37,986 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T09:58:37,987 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T09:58:37,987 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T09:58:37,987 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T09:58:37,988 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T09:58:37,988 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T09:58:37,988 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T09:58:37,989 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T09:58:37,989 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T09:58:37,990 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T09:58:37,990 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T09:58:37,991 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T09:58:37,992 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T09:58:37,992 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T09:58:37,993 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T09:58:37,993 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T09:58:37,994 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T09:58:37,995 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T09:58:37,995 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T09:58:37,996 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T09:58:37,996 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T09:58:37,997 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T09:58:37,998 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T09:58:37,999 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T09:58:37,999 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T09:58:38,000 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T09:58:38,001 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T09:58:38,002 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T09:58:38,002 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T09:58:38,003 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T09:58:38,004 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T09:58:38,005 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T09:58:38,006 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T09:58:38,006 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T09:58:38,007 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T09:58:38,008 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T09:58:38,009 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T09:58:38,010 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T09:58:38,010 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T09:58:38,011 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T09:58:38,012 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T09:58:38,013 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T09:58:38,013 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T09:58:38,014 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T09:58:38,015 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T09:58:38,016 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T09:58:38,017 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T09:58:38,018 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T09:58:38,019 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T09:58:38,020 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T09:58:38,021 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-16T09:58:38,021 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 7c7f87a6124bfe2f4049ee7803e0f98c 3/3 column families, dataSize=53.67 KB heapSize=141.38 KB 2024-11-16T09:58:38,021 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T09:58:38,021 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T09:58:38,022 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T09:58:38,022 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] regionserver.HRegion(8855): Flush requested on 7c7f87a6124bfe2f4049ee7803e0f98c 2024-11-16T09:58:38,023 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T09:58:38,024 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T09:58:38,025 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T09:58:38,026 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T09:58:38,026 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T09:58:38,027 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T09:58:38,028 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T09:58:38,029 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T09:58:38,029 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T09:58:38,030 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T09:58:38,031 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T09:58:38,032 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T09:58:38,033 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T09:58:38,033 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T09:58:38,034 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T09:58:38,035 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T09:58:38,036 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T09:58:38,036 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T09:58:38,039 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T09:58:38,040 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T09:58:38,041 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T09:58:38,042 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T09:58:38,043 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T09:58:38,043 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241116a398894a58d24b12be2a9d8e28ed5e03_7c7f87a6124bfe2f4049ee7803e0f98c is 50, key is test_row_0/A:col10/1731751117995/Put/seqid=0 2024-11-16T09:58:38,044 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T09:58:38,045 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T09:58:38,046 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T09:58:38,046 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T09:58:38,047 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T09:58:38,049 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T09:58:38,050 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T09:58:38,052 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T09:58:38,053 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T09:58:38,054 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T09:58:38,056 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T09:58:38,058 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T09:58:38,058 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T09:58:38,059 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T09:58:38,060 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T09:58:38,067 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T09:58:38,068 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T09:58:38,069 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T09:58:38,070 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T09:58:38,071 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T09:58:38,072 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T09:58:38,073 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T09:58:38,074 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T09:58:38,075 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T09:58:38,076 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T09:58:38,076 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T09:58:38,077 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T09:58:38,078 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T09:58:38,085 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T09:58:38,086 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T09:58:38,087 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T09:58:38,088 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T09:58:38,089 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T09:58:38,090 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T09:58:38,103 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37591 is added to blk_1073741993_1169 (size=24358) 2024-11-16T09:58:38,156 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c7f87a6124bfe2f4049ee7803e0f98c, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T09:58:38,156 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52400 deadline: 1731751178142, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c7f87a6124bfe2f4049ee7803e0f98c, server=037c58a19ddb,37267,1731751077844 2024-11-16T09:58:38,157 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731751108989.7c7f87a6124bfe2f4049ee7803e0f98c., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 , the old value is region=TestAcidGuarantees,,1731751108989.7c7f87a6124bfe2f4049ee7803e0f98c., hostname=037c58a19ddb,37267,1731751077844, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c7f87a6124bfe2f4049ee7803e0f98c, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:58:38,157 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731751108989.7c7f87a6124bfe2f4049ee7803e0f98c., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c7f87a6124bfe2f4049ee7803e0f98c, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:58:38,157 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731751108989.7c7f87a6124bfe2f4049ee7803e0f98c., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 because the exception is null or not the one we care about 2024-11-16T09:58:38,164 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c7f87a6124bfe2f4049ee7803e0f98c, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T09:58:38,164 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52420 deadline: 1731751178155, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c7f87a6124bfe2f4049ee7803e0f98c, server=037c58a19ddb,37267,1731751077844 2024-11-16T09:58:38,165 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731751108989.7c7f87a6124bfe2f4049ee7803e0f98c., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 , the old value is region=TestAcidGuarantees,,1731751108989.7c7f87a6124bfe2f4049ee7803e0f98c., hostname=037c58a19ddb,37267,1731751077844, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c7f87a6124bfe2f4049ee7803e0f98c, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:58:38,165 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731751108989.7c7f87a6124bfe2f4049ee7803e0f98c., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c7f87a6124bfe2f4049ee7803e0f98c, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:58:38,165 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c7f87a6124bfe2f4049ee7803e0f98c, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T09:58:38,165 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731751108989.7c7f87a6124bfe2f4049ee7803e0f98c., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 because the exception is null or not the one we care about 2024-11-16T09:58:38,165 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52396 deadline: 1731751178156, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c7f87a6124bfe2f4049ee7803e0f98c, server=037c58a19ddb,37267,1731751077844 2024-11-16T09:58:38,165 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c7f87a6124bfe2f4049ee7803e0f98c, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T09:58:38,165 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731751108989.7c7f87a6124bfe2f4049ee7803e0f98c., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 , the old value is region=TestAcidGuarantees,,1731751108989.7c7f87a6124bfe2f4049ee7803e0f98c., hostname=037c58a19ddb,37267,1731751077844, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c7f87a6124bfe2f4049ee7803e0f98c, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:58:38,165 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731751108989.7c7f87a6124bfe2f4049ee7803e0f98c., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c7f87a6124bfe2f4049ee7803e0f98c, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:58:38,166 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731751108989.7c7f87a6124bfe2f4049ee7803e0f98c., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 because the exception is null or not the one we care about 2024-11-16T09:58:38,165 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52428 deadline: 1731751178157, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c7f87a6124bfe2f4049ee7803e0f98c, server=037c58a19ddb,37267,1731751077844 2024-11-16T09:58:38,166 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731751108989.7c7f87a6124bfe2f4049ee7803e0f98c., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 , the old value is region=TestAcidGuarantees,,1731751108989.7c7f87a6124bfe2f4049ee7803e0f98c., hostname=037c58a19ddb,37267,1731751077844, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c7f87a6124bfe2f4049ee7803e0f98c, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:58:38,167 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731751108989.7c7f87a6124bfe2f4049ee7803e0f98c., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c7f87a6124bfe2f4049ee7803e0f98c, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:58:38,167 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731751108989.7c7f87a6124bfe2f4049ee7803e0f98c., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 because the exception is null or not the one we care about 2024-11-16T09:58:38,197 DEBUG [RS:0;037c58a19ddb:37267-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/7c7f87a6124bfe2f4049ee7803e0f98c/.tmp/C/f7adaf3740b84fe18c83f162e49403a7 as hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/7c7f87a6124bfe2f4049ee7803e0f98c/C/f7adaf3740b84fe18c83f162e49403a7 2024-11-16T09:58:38,204 INFO [RS:0;037c58a19ddb:37267-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 5 (all) file(s) in 7c7f87a6124bfe2f4049ee7803e0f98c/C of 7c7f87a6124bfe2f4049ee7803e0f98c into f7adaf3740b84fe18c83f162e49403a7(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-16T09:58:38,204 DEBUG [RS:0;037c58a19ddb:37267-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 7c7f87a6124bfe2f4049ee7803e0f98c: 2024-11-16T09:58:38,204 INFO [RS:0;037c58a19ddb:37267-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731751108989.7c7f87a6124bfe2f4049ee7803e0f98c., storeName=7c7f87a6124bfe2f4049ee7803e0f98c/C, priority=11, startTime=1731751117176; duration=0sec 2024-11-16T09:58:38,204 DEBUG [RS:0;037c58a19ddb:37267-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-16T09:58:38,204 DEBUG [RS:0;037c58a19ddb:37267-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 7c7f87a6124bfe2f4049ee7803e0f98c:C 2024-11-16T09:58:38,265 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c7f87a6124bfe2f4049ee7803e0f98c, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T09:58:38,265 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52400 deadline: 1731751178263, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c7f87a6124bfe2f4049ee7803e0f98c, server=037c58a19ddb,37267,1731751077844 2024-11-16T09:58:38,266 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731751108989.7c7f87a6124bfe2f4049ee7803e0f98c., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 , the old value is region=TestAcidGuarantees,,1731751108989.7c7f87a6124bfe2f4049ee7803e0f98c., hostname=037c58a19ddb,37267,1731751077844, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c7f87a6124bfe2f4049ee7803e0f98c, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:58:38,266 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731751108989.7c7f87a6124bfe2f4049ee7803e0f98c., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c7f87a6124bfe2f4049ee7803e0f98c, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:58:38,266 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731751108989.7c7f87a6124bfe2f4049ee7803e0f98c., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 because the exception is null or not the one we care about 2024-11-16T09:58:38,280 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c7f87a6124bfe2f4049ee7803e0f98c, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T09:58:38,281 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52420 deadline: 1731751178273, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c7f87a6124bfe2f4049ee7803e0f98c, server=037c58a19ddb,37267,1731751077844 2024-11-16T09:58:38,281 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731751108989.7c7f87a6124bfe2f4049ee7803e0f98c., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 , the old value is region=TestAcidGuarantees,,1731751108989.7c7f87a6124bfe2f4049ee7803e0f98c., hostname=037c58a19ddb,37267,1731751077844, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c7f87a6124bfe2f4049ee7803e0f98c, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:58:38,281 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731751108989.7c7f87a6124bfe2f4049ee7803e0f98c., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c7f87a6124bfe2f4049ee7803e0f98c, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:58:38,281 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731751108989.7c7f87a6124bfe2f4049ee7803e0f98c., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 because the exception is null or not the one we care about 2024-11-16T09:58:38,281 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c7f87a6124bfe2f4049ee7803e0f98c, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T09:58:38,282 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c7f87a6124bfe2f4049ee7803e0f98c, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T09:58:38,282 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52396 deadline: 1731751178273, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c7f87a6124bfe2f4049ee7803e0f98c, server=037c58a19ddb,37267,1731751077844 2024-11-16T09:58:38,282 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52428 deadline: 1731751178273, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c7f87a6124bfe2f4049ee7803e0f98c, server=037c58a19ddb,37267,1731751077844 2024-11-16T09:58:38,282 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731751108989.7c7f87a6124bfe2f4049ee7803e0f98c., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 , the old value is region=TestAcidGuarantees,,1731751108989.7c7f87a6124bfe2f4049ee7803e0f98c., hostname=037c58a19ddb,37267,1731751077844, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c7f87a6124bfe2f4049ee7803e0f98c, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:58:38,282 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731751108989.7c7f87a6124bfe2f4049ee7803e0f98c., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c7f87a6124bfe2f4049ee7803e0f98c, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:58:38,282 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731751108989.7c7f87a6124bfe2f4049ee7803e0f98c., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 because the exception is null or not the one we care about 2024-11-16T09:58:38,283 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731751108989.7c7f87a6124bfe2f4049ee7803e0f98c., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 , the old value is region=TestAcidGuarantees,,1731751108989.7c7f87a6124bfe2f4049ee7803e0f98c., hostname=037c58a19ddb,37267,1731751077844, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c7f87a6124bfe2f4049ee7803e0f98c, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:58:38,283 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731751108989.7c7f87a6124bfe2f4049ee7803e0f98c., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c7f87a6124bfe2f4049ee7803e0f98c, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:58:38,283 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731751108989.7c7f87a6124bfe2f4049ee7803e0f98c., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 because the exception is null or not the one we care about 2024-11-16T09:58:38,474 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c7f87a6124bfe2f4049ee7803e0f98c, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T09:58:38,474 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52400 deadline: 1731751178473, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c7f87a6124bfe2f4049ee7803e0f98c, server=037c58a19ddb,37267,1731751077844 2024-11-16T09:58:38,474 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731751108989.7c7f87a6124bfe2f4049ee7803e0f98c., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 , the old value is region=TestAcidGuarantees,,1731751108989.7c7f87a6124bfe2f4049ee7803e0f98c., hostname=037c58a19ddb,37267,1731751077844, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c7f87a6124bfe2f4049ee7803e0f98c, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:58:38,474 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731751108989.7c7f87a6124bfe2f4049ee7803e0f98c., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c7f87a6124bfe2f4049ee7803e0f98c, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:58:38,475 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731751108989.7c7f87a6124bfe2f4049ee7803e0f98c., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 because the exception is null or not the one we care about 2024-11-16T09:58:38,476 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37877 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=40 2024-11-16T09:58:38,479 INFO [RPCClient-NioEventLoopGroup-4-7 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-16T09:58:38,480 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37877 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-16T09:58:38,482 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37877 {}] procedure2.ProcedureExecutor(1139): Stored pid=42, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=42, table=TestAcidGuarantees 2024-11-16T09:58:38,483 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37877 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=42 2024-11-16T09:58:38,483 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=42, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=42, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-16T09:58:38,484 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=42, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=42, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-16T09:58:38,484 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=43, ppid=42, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-16T09:58:38,497 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c7f87a6124bfe2f4049ee7803e0f98c, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T09:58:38,497 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52420 deadline: 1731751178493, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c7f87a6124bfe2f4049ee7803e0f98c, server=037c58a19ddb,37267,1731751077844 2024-11-16T09:58:38,498 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731751108989.7c7f87a6124bfe2f4049ee7803e0f98c., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 , the old value is region=TestAcidGuarantees,,1731751108989.7c7f87a6124bfe2f4049ee7803e0f98c., hostname=037c58a19ddb,37267,1731751077844, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c7f87a6124bfe2f4049ee7803e0f98c, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:58:38,498 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731751108989.7c7f87a6124bfe2f4049ee7803e0f98c., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c7f87a6124bfe2f4049ee7803e0f98c, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:58:38,498 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731751108989.7c7f87a6124bfe2f4049ee7803e0f98c., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 because the exception is null or not the one we care about 2024-11-16T09:58:38,498 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c7f87a6124bfe2f4049ee7803e0f98c, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T09:58:38,498 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52396 deadline: 1731751178493, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c7f87a6124bfe2f4049ee7803e0f98c, server=037c58a19ddb,37267,1731751077844 2024-11-16T09:58:38,498 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731751108989.7c7f87a6124bfe2f4049ee7803e0f98c., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 , the old value is region=TestAcidGuarantees,,1731751108989.7c7f87a6124bfe2f4049ee7803e0f98c., hostname=037c58a19ddb,37267,1731751077844, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c7f87a6124bfe2f4049ee7803e0f98c, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:58:38,499 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731751108989.7c7f87a6124bfe2f4049ee7803e0f98c., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c7f87a6124bfe2f4049ee7803e0f98c, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:58:38,499 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731751108989.7c7f87a6124bfe2f4049ee7803e0f98c., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 because the exception is null or not the one we care about 2024-11-16T09:58:38,499 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c7f87a6124bfe2f4049ee7803e0f98c, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T09:58:38,499 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52428 deadline: 1731751178493, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c7f87a6124bfe2f4049ee7803e0f98c, server=037c58a19ddb,37267,1731751077844 2024-11-16T09:58:38,499 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731751108989.7c7f87a6124bfe2f4049ee7803e0f98c., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 , the old value is region=TestAcidGuarantees,,1731751108989.7c7f87a6124bfe2f4049ee7803e0f98c., hostname=037c58a19ddb,37267,1731751077844, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c7f87a6124bfe2f4049ee7803e0f98c, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:58:38,499 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731751108989.7c7f87a6124bfe2f4049ee7803e0f98c., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c7f87a6124bfe2f4049ee7803e0f98c, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:58:38,499 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731751108989.7c7f87a6124bfe2f4049ee7803e0f98c., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 because the exception is null or not the one we care about 2024-11-16T09:58:38,505 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T09:58:38,511 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241116a398894a58d24b12be2a9d8e28ed5e03_7c7f87a6124bfe2f4049ee7803e0f98c to hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241116a398894a58d24b12be2a9d8e28ed5e03_7c7f87a6124bfe2f4049ee7803e0f98c 2024-11-16T09:58:38,512 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/7c7f87a6124bfe2f4049ee7803e0f98c/.tmp/A/d69fd5bf934e49ad99e55960bda383ae, store: [table=TestAcidGuarantees family=A region=7c7f87a6124bfe2f4049ee7803e0f98c] 2024-11-16T09:58:38,513 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/7c7f87a6124bfe2f4049ee7803e0f98c/.tmp/A/d69fd5bf934e49ad99e55960bda383ae is 175, key is test_row_0/A:col10/1731751117995/Put/seqid=0 2024-11-16T09:58:38,517 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37591 is added to blk_1073741994_1170 (size=73995) 2024-11-16T09:58:38,519 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=103, memsize=17.9 K, hasBloomFilter=true, into tmp file hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/7c7f87a6124bfe2f4049ee7803e0f98c/.tmp/A/d69fd5bf934e49ad99e55960bda383ae 2024-11-16T09:58:38,538 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/7c7f87a6124bfe2f4049ee7803e0f98c/.tmp/B/84b81a5e733b45f88388e7fc5aeae30f is 50, key is test_row_0/B:col10/1731751117995/Put/seqid=0 2024-11-16T09:58:38,586 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37591 is added to blk_1073741995_1171 (size=12001) 2024-11-16T09:58:38,588 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=103 (bloomFilter=true), to=hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/7c7f87a6124bfe2f4049ee7803e0f98c/.tmp/B/84b81a5e733b45f88388e7fc5aeae30f 2024-11-16T09:58:38,593 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37877 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=42 2024-11-16T09:58:38,602 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/7c7f87a6124bfe2f4049ee7803e0f98c/.tmp/C/821acc26a10c434aad651ccdb01cfe0a is 50, key is test_row_0/C:col10/1731751117995/Put/seqid=0 2024-11-16T09:58:38,613 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37591 is added to blk_1073741996_1172 (size=12001) 2024-11-16T09:58:38,621 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=103 (bloomFilter=true), to=hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/7c7f87a6124bfe2f4049ee7803e0f98c/.tmp/C/821acc26a10c434aad651ccdb01cfe0a 2024-11-16T09:58:38,628 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/7c7f87a6124bfe2f4049ee7803e0f98c/.tmp/A/d69fd5bf934e49ad99e55960bda383ae as hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/7c7f87a6124bfe2f4049ee7803e0f98c/A/d69fd5bf934e49ad99e55960bda383ae 2024-11-16T09:58:38,632 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/7c7f87a6124bfe2f4049ee7803e0f98c/A/d69fd5bf934e49ad99e55960bda383ae, entries=400, sequenceid=103, filesize=72.3 K 2024-11-16T09:58:38,633 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/7c7f87a6124bfe2f4049ee7803e0f98c/.tmp/B/84b81a5e733b45f88388e7fc5aeae30f as hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/7c7f87a6124bfe2f4049ee7803e0f98c/B/84b81a5e733b45f88388e7fc5aeae30f 2024-11-16T09:58:38,638 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=37267 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=43 2024-11-16T09:58:38,638 DEBUG [RS_FLUSH_OPERATIONS-regionserver/037c58a19ddb:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731751108989.7c7f87a6124bfe2f4049ee7803e0f98c. 2024-11-16T09:58:38,638 DEBUG [RS_FLUSH_OPERATIONS-regionserver/037c58a19ddb:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731751108989.7c7f87a6124bfe2f4049ee7803e0f98c. as already flushing 2024-11-16T09:58:38,638 DEBUG [RS_FLUSH_OPERATIONS-regionserver/037c58a19ddb:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731751108989.7c7f87a6124bfe2f4049ee7803e0f98c. 2024-11-16T09:58:38,638 ERROR [RS_FLUSH_OPERATIONS-regionserver/037c58a19ddb:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] handler.RSProcedureHandler(58): pid=43 java.io.IOException: Unable to complete flush {ENCODED => 7c7f87a6124bfe2f4049ee7803e0f98c, NAME => 'TestAcidGuarantees,,1731751108989.7c7f87a6124bfe2f4049ee7803e0f98c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T09:58:38,638 DEBUG [RS_FLUSH_OPERATIONS-regionserver/037c58a19ddb:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=43 java.io.IOException: Unable to complete flush {ENCODED => 7c7f87a6124bfe2f4049ee7803e0f98c, NAME => 'TestAcidGuarantees,,1731751108989.7c7f87a6124bfe2f4049ee7803e0f98c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T09:58:38,640 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/7c7f87a6124bfe2f4049ee7803e0f98c/B/84b81a5e733b45f88388e7fc5aeae30f, entries=150, sequenceid=103, filesize=11.7 K 2024-11-16T09:58:38,641 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/7c7f87a6124bfe2f4049ee7803e0f98c/.tmp/C/821acc26a10c434aad651ccdb01cfe0a as hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/7c7f87a6124bfe2f4049ee7803e0f98c/C/821acc26a10c434aad651ccdb01cfe0a 2024-11-16T09:58:38,641 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37877 {}] master.HMaster(4177): Remote procedure failed, pid=43 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 7c7f87a6124bfe2f4049ee7803e0f98c, NAME => 'TestAcidGuarantees,,1731751108989.7c7f87a6124bfe2f4049ee7803e0f98c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 7c7f87a6124bfe2f4049ee7803e0f98c, NAME => 'TestAcidGuarantees,,1731751108989.7c7f87a6124bfe2f4049ee7803e0f98c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T09:58:38,648 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/7c7f87a6124bfe2f4049ee7803e0f98c/C/821acc26a10c434aad651ccdb01cfe0a, entries=150, sequenceid=103, filesize=11.7 K 2024-11-16T09:58:38,650 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~141.33 KB/144720, currentSize=154.31 KB/158010 for 7c7f87a6124bfe2f4049ee7803e0f98c in 628ms, sequenceid=103, compaction requested=true 2024-11-16T09:58:38,650 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 7c7f87a6124bfe2f4049ee7803e0f98c: 2024-11-16T09:58:38,650 DEBUG [RS:0;037c58a19ddb:37267-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-16T09:58:38,650 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 7c7f87a6124bfe2f4049ee7803e0f98c:A, priority=-2147483648, current under compaction store size is 1 2024-11-16T09:58:38,651 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-16T09:58:38,651 DEBUG [RS:0;037c58a19ddb:37267-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 136043 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-16T09:58:38,651 DEBUG [RS:0;037c58a19ddb:37267-shortCompactions-0 {}] regionserver.HStore(1541): 7c7f87a6124bfe2f4049ee7803e0f98c/A is initiating minor compaction (all files) 2024-11-16T09:58:38,651 INFO [RS:0;037c58a19ddb:37267-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 7c7f87a6124bfe2f4049ee7803e0f98c/A in TestAcidGuarantees,,1731751108989.7c7f87a6124bfe2f4049ee7803e0f98c. 2024-11-16T09:58:38,651 INFO [RS:0;037c58a19ddb:37267-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/7c7f87a6124bfe2f4049ee7803e0f98c/A/decd818193a64048853aa6c79bdf0976, hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/7c7f87a6124bfe2f4049ee7803e0f98c/A/d4eba6854cf24b93ab8094b32b349735, hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/7c7f87a6124bfe2f4049ee7803e0f98c/A/d69fd5bf934e49ad99e55960bda383ae] into tmpdir=hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/7c7f87a6124bfe2f4049ee7803e0f98c/.tmp, totalSize=132.9 K 2024-11-16T09:58:38,651 INFO [RS:0;037c58a19ddb:37267-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731751108989.7c7f87a6124bfe2f4049ee7803e0f98c. 2024-11-16T09:58:38,651 DEBUG [RS:0;037c58a19ddb:37267-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731751108989.7c7f87a6124bfe2f4049ee7803e0f98c. files: [hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/7c7f87a6124bfe2f4049ee7803e0f98c/A/decd818193a64048853aa6c79bdf0976, hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/7c7f87a6124bfe2f4049ee7803e0f98c/A/d4eba6854cf24b93ab8094b32b349735, hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/7c7f87a6124bfe2f4049ee7803e0f98c/A/d69fd5bf934e49ad99e55960bda383ae] 2024-11-16T09:58:38,652 DEBUG [RS:0;037c58a19ddb:37267-shortCompactions-0 {}] compactions.Compactor(225): Compacting decd818193a64048853aa6c79bdf0976, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=79, earliestPutTs=1731751114599 2024-11-16T09:58:38,652 DEBUG [RS:0;037c58a19ddb:37267-shortCompactions-0 {}] compactions.Compactor(225): Compacting d4eba6854cf24b93ab8094b32b349735, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=90, earliestPutTs=1731751115756 2024-11-16T09:58:38,653 DEBUG [RS:0;037c58a19ddb:37267-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-16T09:58:38,653 DEBUG [RS:0;037c58a19ddb:37267-shortCompactions-0 {}] compactions.Compactor(225): Compacting d69fd5bf934e49ad99e55960bda383ae, keycount=400, bloomtype=ROW, size=72.3 K, encoding=NONE, compression=NONE, seqNum=103, earliestPutTs=1731751117992 2024-11-16T09:58:38,654 DEBUG [RS:0;037c58a19ddb:37267-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36141 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-16T09:58:38,654 DEBUG [RS:0;037c58a19ddb:37267-longCompactions-0 {}] regionserver.HStore(1541): 7c7f87a6124bfe2f4049ee7803e0f98c/B is initiating minor compaction (all files) 2024-11-16T09:58:38,655 INFO [RS:0;037c58a19ddb:37267-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 7c7f87a6124bfe2f4049ee7803e0f98c/B in TestAcidGuarantees,,1731751108989.7c7f87a6124bfe2f4049ee7803e0f98c. 2024-11-16T09:58:38,655 INFO [RS:0;037c58a19ddb:37267-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/7c7f87a6124bfe2f4049ee7803e0f98c/B/b471d622e7e145b989bbb16c6e434c3c, hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/7c7f87a6124bfe2f4049ee7803e0f98c/B/85b3c54b164145d2925f3a8bb12ce8fe, hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/7c7f87a6124bfe2f4049ee7803e0f98c/B/84b81a5e733b45f88388e7fc5aeae30f] into tmpdir=hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/7c7f87a6124bfe2f4049ee7803e0f98c/.tmp, totalSize=35.3 K 2024-11-16T09:58:38,655 DEBUG [RS:0;037c58a19ddb:37267-longCompactions-0 {}] compactions.Compactor(225): Compacting b471d622e7e145b989bbb16c6e434c3c, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=79, earliestPutTs=1731751114599 2024-11-16T09:58:38,656 DEBUG [RS:0;037c58a19ddb:37267-longCompactions-0 {}] compactions.Compactor(225): Compacting 85b3c54b164145d2925f3a8bb12ce8fe, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=90, earliestPutTs=1731751115756 2024-11-16T09:58:38,657 DEBUG [RS:0;037c58a19ddb:37267-longCompactions-0 {}] compactions.Compactor(225): Compacting 84b81a5e733b45f88388e7fc5aeae30f, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=103, earliestPutTs=1731751117995 2024-11-16T09:58:38,661 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 7c7f87a6124bfe2f4049ee7803e0f98c:B, priority=-2147483648, current under compaction store size is 2 2024-11-16T09:58:38,661 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-16T09:58:38,661 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 7c7f87a6124bfe2f4049ee7803e0f98c:C, priority=-2147483648, current under compaction store size is 3 2024-11-16T09:58:38,661 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-16T09:58:38,663 INFO [RS:0;037c58a19ddb:37267-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=7c7f87a6124bfe2f4049ee7803e0f98c] 2024-11-16T09:58:38,668 DEBUG [RS:0;037c58a19ddb:37267-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e202411166e71b19bcfac48e2b572b65adc0b9094_7c7f87a6124bfe2f4049ee7803e0f98c store=[table=TestAcidGuarantees family=A region=7c7f87a6124bfe2f4049ee7803e0f98c] 2024-11-16T09:58:38,670 INFO [RS:0;037c58a19ddb:37267-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 7c7f87a6124bfe2f4049ee7803e0f98c#B#compaction#151 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-16T09:58:38,671 DEBUG [RS:0;037c58a19ddb:37267-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/7c7f87a6124bfe2f4049ee7803e0f98c/.tmp/B/61c3f1d80f724fefaa1bfbedddebbb01 is 50, key is test_row_0/B:col10/1731751117995/Put/seqid=0 2024-11-16T09:58:38,672 DEBUG [RS:0;037c58a19ddb:37267-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e202411166e71b19bcfac48e2b572b65adc0b9094_7c7f87a6124bfe2f4049ee7803e0f98c, store=[table=TestAcidGuarantees family=A region=7c7f87a6124bfe2f4049ee7803e0f98c] 2024-11-16T09:58:38,672 DEBUG [RS:0;037c58a19ddb:37267-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411166e71b19bcfac48e2b572b65adc0b9094_7c7f87a6124bfe2f4049ee7803e0f98c because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=7c7f87a6124bfe2f4049ee7803e0f98c] 2024-11-16T09:58:38,691 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37591 is added to blk_1073741997_1173 (size=12241) 2024-11-16T09:58:38,718 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37591 is added to blk_1073741998_1174 (size=4469) 2024-11-16T09:58:38,720 INFO [RS:0;037c58a19ddb:37267-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 7c7f87a6124bfe2f4049ee7803e0f98c#A#compaction#150 average throughput is 0.43 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-16T09:58:38,721 DEBUG [RS:0;037c58a19ddb:37267-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/7c7f87a6124bfe2f4049ee7803e0f98c/.tmp/A/4ca189b9a842483db6e475afcc8899aa is 175, key is test_row_0/A:col10/1731751117995/Put/seqid=0 2024-11-16T09:58:38,745 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37591 is added to blk_1073741999_1175 (size=31195) 2024-11-16T09:58:38,788 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] regionserver.HRegion(8855): Flush requested on 7c7f87a6124bfe2f4049ee7803e0f98c 2024-11-16T09:58:38,788 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-16T09:58:38,788 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 7c7f87a6124bfe2f4049ee7803e0f98c 3/3 column families, dataSize=161.02 KB heapSize=422.63 KB 2024-11-16T09:58:38,794 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=37267 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=43 2024-11-16T09:58:38,794 DEBUG [RS_FLUSH_OPERATIONS-regionserver/037c58a19ddb:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731751108989.7c7f87a6124bfe2f4049ee7803e0f98c. 2024-11-16T09:58:38,794 DEBUG [RS_FLUSH_OPERATIONS-regionserver/037c58a19ddb:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731751108989.7c7f87a6124bfe2f4049ee7803e0f98c. as already flushing 2024-11-16T09:58:38,794 DEBUG [RS_FLUSH_OPERATIONS-regionserver/037c58a19ddb:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731751108989.7c7f87a6124bfe2f4049ee7803e0f98c. 2024-11-16T09:58:38,794 ERROR [RS_FLUSH_OPERATIONS-regionserver/037c58a19ddb:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] handler.RSProcedureHandler(58): pid=43 java.io.IOException: Unable to complete flush {ENCODED => 7c7f87a6124bfe2f4049ee7803e0f98c, NAME => 'TestAcidGuarantees,,1731751108989.7c7f87a6124bfe2f4049ee7803e0f98c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T09:58:38,795 DEBUG [RS_FLUSH_OPERATIONS-regionserver/037c58a19ddb:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=43 java.io.IOException: Unable to complete flush {ENCODED => 7c7f87a6124bfe2f4049ee7803e0f98c, NAME => 'TestAcidGuarantees,,1731751108989.7c7f87a6124bfe2f4049ee7803e0f98c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T09:58:38,796 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37877 {}] master.HMaster(4177): Remote procedure failed, pid=43 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 7c7f87a6124bfe2f4049ee7803e0f98c, NAME => 'TestAcidGuarantees,,1731751108989.7c7f87a6124bfe2f4049ee7803e0f98c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 7c7f87a6124bfe2f4049ee7803e0f98c, NAME => 'TestAcidGuarantees,,1731751108989.7c7f87a6124bfe2f4049ee7803e0f98c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T09:58:38,804 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37877 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=42 2024-11-16T09:58:38,808 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c7f87a6124bfe2f4049ee7803e0f98c, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T09:58:38,808 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52428 deadline: 1731751178805, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c7f87a6124bfe2f4049ee7803e0f98c, server=037c58a19ddb,37267,1731751077844 2024-11-16T09:58:38,809 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731751108989.7c7f87a6124bfe2f4049ee7803e0f98c., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 , the old value is region=TestAcidGuarantees,,1731751108989.7c7f87a6124bfe2f4049ee7803e0f98c., hostname=037c58a19ddb,37267,1731751077844, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c7f87a6124bfe2f4049ee7803e0f98c, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:58:38,809 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731751108989.7c7f87a6124bfe2f4049ee7803e0f98c., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c7f87a6124bfe2f4049ee7803e0f98c, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:58:38,809 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731751108989.7c7f87a6124bfe2f4049ee7803e0f98c., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 because the exception is null or not the one we care about 2024-11-16T09:58:38,813 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c7f87a6124bfe2f4049ee7803e0f98c, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T09:58:38,813 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52392 deadline: 1731751178807, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c7f87a6124bfe2f4049ee7803e0f98c, server=037c58a19ddb,37267,1731751077844 2024-11-16T09:58:38,813 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c7f87a6124bfe2f4049ee7803e0f98c, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T09:58:38,813 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52400 deadline: 1731751178807, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c7f87a6124bfe2f4049ee7803e0f98c, server=037c58a19ddb,37267,1731751077844 2024-11-16T09:58:38,813 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731751108989.7c7f87a6124bfe2f4049ee7803e0f98c., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 , the old value is region=TestAcidGuarantees,,1731751108989.7c7f87a6124bfe2f4049ee7803e0f98c., hostname=037c58a19ddb,37267,1731751077844, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c7f87a6124bfe2f4049ee7803e0f98c, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:58:38,813 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731751108989.7c7f87a6124bfe2f4049ee7803e0f98c., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c7f87a6124bfe2f4049ee7803e0f98c, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:58:38,813 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731751108989.7c7f87a6124bfe2f4049ee7803e0f98c., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 because the exception is null or not the one we care about 2024-11-16T09:58:38,814 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731751108989.7c7f87a6124bfe2f4049ee7803e0f98c., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 , the old value is region=TestAcidGuarantees,,1731751108989.7c7f87a6124bfe2f4049ee7803e0f98c., hostname=037c58a19ddb,37267,1731751077844, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c7f87a6124bfe2f4049ee7803e0f98c, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:58:38,814 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731751108989.7c7f87a6124bfe2f4049ee7803e0f98c., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c7f87a6124bfe2f4049ee7803e0f98c, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:58:38,814 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731751108989.7c7f87a6124bfe2f4049ee7803e0f98c., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 because the exception is null or not the one we care about 2024-11-16T09:58:38,814 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c7f87a6124bfe2f4049ee7803e0f98c, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T09:58:38,814 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52396 deadline: 1731751178809, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c7f87a6124bfe2f4049ee7803e0f98c, server=037c58a19ddb,37267,1731751077844 2024-11-16T09:58:38,814 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c7f87a6124bfe2f4049ee7803e0f98c, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T09:58:38,814 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731751108989.7c7f87a6124bfe2f4049ee7803e0f98c., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 , the old value is region=TestAcidGuarantees,,1731751108989.7c7f87a6124bfe2f4049ee7803e0f98c., hostname=037c58a19ddb,37267,1731751077844, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c7f87a6124bfe2f4049ee7803e0f98c, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:58:38,814 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52420 deadline: 1731751178809, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c7f87a6124bfe2f4049ee7803e0f98c, server=037c58a19ddb,37267,1731751077844 2024-11-16T09:58:38,814 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731751108989.7c7f87a6124bfe2f4049ee7803e0f98c., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c7f87a6124bfe2f4049ee7803e0f98c, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:58:38,814 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731751108989.7c7f87a6124bfe2f4049ee7803e0f98c., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 because the exception is null or not the one we care about 2024-11-16T09:58:38,815 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731751108989.7c7f87a6124bfe2f4049ee7803e0f98c., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 , the old value is region=TestAcidGuarantees,,1731751108989.7c7f87a6124bfe2f4049ee7803e0f98c., hostname=037c58a19ddb,37267,1731751077844, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c7f87a6124bfe2f4049ee7803e0f98c, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:58:38,815 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731751108989.7c7f87a6124bfe2f4049ee7803e0f98c., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c7f87a6124bfe2f4049ee7803e0f98c, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:58:38,815 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731751108989.7c7f87a6124bfe2f4049ee7803e0f98c., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 because the exception is null or not the one we care about 2024-11-16T09:58:38,823 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241116e32f3ab2cc8743e18287927778b733e6_7c7f87a6124bfe2f4049ee7803e0f98c is 50, key is test_row_0/A:col10/1731751118154/Put/seqid=0 2024-11-16T09:58:38,841 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37591 is added to blk_1073742000_1176 (size=12204) 2024-11-16T09:58:38,925 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c7f87a6124bfe2f4049ee7803e0f98c, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T09:58:38,925 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52396 deadline: 1731751178924, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c7f87a6124bfe2f4049ee7803e0f98c, server=037c58a19ddb,37267,1731751077844 2024-11-16T09:58:38,925 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c7f87a6124bfe2f4049ee7803e0f98c, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T09:58:38,925 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731751108989.7c7f87a6124bfe2f4049ee7803e0f98c., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 , the old value is region=TestAcidGuarantees,,1731751108989.7c7f87a6124bfe2f4049ee7803e0f98c., hostname=037c58a19ddb,37267,1731751077844, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c7f87a6124bfe2f4049ee7803e0f98c, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:58:38,925 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52392 deadline: 1731751178924, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c7f87a6124bfe2f4049ee7803e0f98c, server=037c58a19ddb,37267,1731751077844 2024-11-16T09:58:38,925 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731751108989.7c7f87a6124bfe2f4049ee7803e0f98c., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c7f87a6124bfe2f4049ee7803e0f98c, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:58:38,926 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731751108989.7c7f87a6124bfe2f4049ee7803e0f98c., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 because the exception is null or not the one we care about 2024-11-16T09:58:38,926 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c7f87a6124bfe2f4049ee7803e0f98c, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T09:58:38,926 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731751108989.7c7f87a6124bfe2f4049ee7803e0f98c., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 , the old value is region=TestAcidGuarantees,,1731751108989.7c7f87a6124bfe2f4049ee7803e0f98c., hostname=037c58a19ddb,37267,1731751077844, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c7f87a6124bfe2f4049ee7803e0f98c, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:58:38,926 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731751108989.7c7f87a6124bfe2f4049ee7803e0f98c., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c7f87a6124bfe2f4049ee7803e0f98c, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:58:38,926 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52400 deadline: 1731751178924, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c7f87a6124bfe2f4049ee7803e0f98c, server=037c58a19ddb,37267,1731751077844 2024-11-16T09:58:38,926 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731751108989.7c7f87a6124bfe2f4049ee7803e0f98c., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 because the exception is null or not the one we care about 2024-11-16T09:58:38,926 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c7f87a6124bfe2f4049ee7803e0f98c, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T09:58:38,926 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731751108989.7c7f87a6124bfe2f4049ee7803e0f98c., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 , the old value is region=TestAcidGuarantees,,1731751108989.7c7f87a6124bfe2f4049ee7803e0f98c., hostname=037c58a19ddb,37267,1731751077844, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c7f87a6124bfe2f4049ee7803e0f98c, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:58:38,926 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731751108989.7c7f87a6124bfe2f4049ee7803e0f98c., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c7f87a6124bfe2f4049ee7803e0f98c, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:58:38,926 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52420 deadline: 1731751178924, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c7f87a6124bfe2f4049ee7803e0f98c, server=037c58a19ddb,37267,1731751077844 2024-11-16T09:58:38,926 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731751108989.7c7f87a6124bfe2f4049ee7803e0f98c., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 because the exception is null or not the one we care about 2024-11-16T09:58:38,927 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731751108989.7c7f87a6124bfe2f4049ee7803e0f98c., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 , the old value is region=TestAcidGuarantees,,1731751108989.7c7f87a6124bfe2f4049ee7803e0f98c., hostname=037c58a19ddb,37267,1731751077844, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c7f87a6124bfe2f4049ee7803e0f98c, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:58:38,927 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731751108989.7c7f87a6124bfe2f4049ee7803e0f98c., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c7f87a6124bfe2f4049ee7803e0f98c, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:58:38,927 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731751108989.7c7f87a6124bfe2f4049ee7803e0f98c., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 because the exception is null or not the one we care about 2024-11-16T09:58:38,948 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=37267 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=43 2024-11-16T09:58:38,948 DEBUG [RS_FLUSH_OPERATIONS-regionserver/037c58a19ddb:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731751108989.7c7f87a6124bfe2f4049ee7803e0f98c. 2024-11-16T09:58:38,948 DEBUG [RS_FLUSH_OPERATIONS-regionserver/037c58a19ddb:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731751108989.7c7f87a6124bfe2f4049ee7803e0f98c. as already flushing 2024-11-16T09:58:38,948 DEBUG [RS_FLUSH_OPERATIONS-regionserver/037c58a19ddb:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731751108989.7c7f87a6124bfe2f4049ee7803e0f98c. 2024-11-16T09:58:38,948 ERROR [RS_FLUSH_OPERATIONS-regionserver/037c58a19ddb:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] handler.RSProcedureHandler(58): pid=43 java.io.IOException: Unable to complete flush {ENCODED => 7c7f87a6124bfe2f4049ee7803e0f98c, NAME => 'TestAcidGuarantees,,1731751108989.7c7f87a6124bfe2f4049ee7803e0f98c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T09:58:38,949 DEBUG [RS_FLUSH_OPERATIONS-regionserver/037c58a19ddb:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=43 java.io.IOException: Unable to complete flush {ENCODED => 7c7f87a6124bfe2f4049ee7803e0f98c, NAME => 'TestAcidGuarantees,,1731751108989.7c7f87a6124bfe2f4049ee7803e0f98c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T09:58:38,949 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37877 {}] master.HMaster(4177): Remote procedure failed, pid=43 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 7c7f87a6124bfe2f4049ee7803e0f98c, NAME => 'TestAcidGuarantees,,1731751108989.7c7f87a6124bfe2f4049ee7803e0f98c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 7c7f87a6124bfe2f4049ee7803e0f98c, NAME => 'TestAcidGuarantees,,1731751108989.7c7f87a6124bfe2f4049ee7803e0f98c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T09:58:39,099 DEBUG [RS:0;037c58a19ddb:37267-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/7c7f87a6124bfe2f4049ee7803e0f98c/.tmp/B/61c3f1d80f724fefaa1bfbedddebbb01 as hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/7c7f87a6124bfe2f4049ee7803e0f98c/B/61c3f1d80f724fefaa1bfbedddebbb01 2024-11-16T09:58:39,103 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=37267 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=43 2024-11-16T09:58:39,103 DEBUG [RS_FLUSH_OPERATIONS-regionserver/037c58a19ddb:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731751108989.7c7f87a6124bfe2f4049ee7803e0f98c. 2024-11-16T09:58:39,103 DEBUG [RS_FLUSH_OPERATIONS-regionserver/037c58a19ddb:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731751108989.7c7f87a6124bfe2f4049ee7803e0f98c. as already flushing 2024-11-16T09:58:39,104 DEBUG [RS_FLUSH_OPERATIONS-regionserver/037c58a19ddb:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731751108989.7c7f87a6124bfe2f4049ee7803e0f98c. 2024-11-16T09:58:39,104 ERROR [RS_FLUSH_OPERATIONS-regionserver/037c58a19ddb:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] handler.RSProcedureHandler(58): pid=43 java.io.IOException: Unable to complete flush {ENCODED => 7c7f87a6124bfe2f4049ee7803e0f98c, NAME => 'TestAcidGuarantees,,1731751108989.7c7f87a6124bfe2f4049ee7803e0f98c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T09:58:39,104 DEBUG [RS_FLUSH_OPERATIONS-regionserver/037c58a19ddb:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=43 java.io.IOException: Unable to complete flush {ENCODED => 7c7f87a6124bfe2f4049ee7803e0f98c, NAME => 'TestAcidGuarantees,,1731751108989.7c7f87a6124bfe2f4049ee7803e0f98c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T09:58:39,105 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37877 {}] master.HMaster(4177): Remote procedure failed, pid=43 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 7c7f87a6124bfe2f4049ee7803e0f98c, NAME => 'TestAcidGuarantees,,1731751108989.7c7f87a6124bfe2f4049ee7803e0f98c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 7c7f87a6124bfe2f4049ee7803e0f98c, NAME => 'TestAcidGuarantees,,1731751108989.7c7f87a6124bfe2f4049ee7803e0f98c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T09:58:39,106 INFO [RS:0;037c58a19ddb:37267-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 7c7f87a6124bfe2f4049ee7803e0f98c/B of 7c7f87a6124bfe2f4049ee7803e0f98c into 61c3f1d80f724fefaa1bfbedddebbb01(size=12.0 K), total size for store is 12.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-16T09:58:39,106 DEBUG [RS:0;037c58a19ddb:37267-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 7c7f87a6124bfe2f4049ee7803e0f98c: 2024-11-16T09:58:39,106 INFO [RS:0;037c58a19ddb:37267-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731751108989.7c7f87a6124bfe2f4049ee7803e0f98c., storeName=7c7f87a6124bfe2f4049ee7803e0f98c/B, priority=13, startTime=1731751118653; duration=0sec 2024-11-16T09:58:39,106 DEBUG [RS:0;037c58a19ddb:37267-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-16T09:58:39,106 DEBUG [RS:0;037c58a19ddb:37267-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 7c7f87a6124bfe2f4049ee7803e0f98c:B 2024-11-16T09:58:39,106 DEBUG [RS:0;037c58a19ddb:37267-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 2 store files, 0 compacting, 2 eligible, 16 blocking 2024-11-16T09:58:39,107 DEBUG [RS:0;037c58a19ddb:37267-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 0 files of size 0 starting at candidate #-1 after considering 0 permutations with 0 in ratio 2024-11-16T09:58:39,108 DEBUG [RS:0;037c58a19ddb:37267-longCompactions-0 {}] compactions.SortedCompactionPolicy(232): Not compacting files because we only have 0 files ready for compaction. Need 3 to initiate. 2024-11-16T09:58:39,108 DEBUG [RS:0;037c58a19ddb:37267-longCompactions-0 {}] regionserver.CompactSplit(450): Not compacting TestAcidGuarantees,,1731751108989.7c7f87a6124bfe2f4049ee7803e0f98c. because compaction request was cancelled 2024-11-16T09:58:39,108 DEBUG [RS:0;037c58a19ddb:37267-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 7c7f87a6124bfe2f4049ee7803e0f98c:C 2024-11-16T09:58:39,113 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37877 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=42 2024-11-16T09:58:39,134 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c7f87a6124bfe2f4049ee7803e0f98c, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T09:58:39,134 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52396 deadline: 1731751179133, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c7f87a6124bfe2f4049ee7803e0f98c, server=037c58a19ddb,37267,1731751077844 2024-11-16T09:58:39,134 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c7f87a6124bfe2f4049ee7803e0f98c, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T09:58:39,134 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731751108989.7c7f87a6124bfe2f4049ee7803e0f98c., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 , the old value is region=TestAcidGuarantees,,1731751108989.7c7f87a6124bfe2f4049ee7803e0f98c., hostname=037c58a19ddb,37267,1731751077844, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c7f87a6124bfe2f4049ee7803e0f98c, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:58:39,134 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52392 deadline: 1731751179133, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c7f87a6124bfe2f4049ee7803e0f98c, server=037c58a19ddb,37267,1731751077844 2024-11-16T09:58:39,134 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731751108989.7c7f87a6124bfe2f4049ee7803e0f98c., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c7f87a6124bfe2f4049ee7803e0f98c, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:58:39,135 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731751108989.7c7f87a6124bfe2f4049ee7803e0f98c., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 because the exception is null or not the one we care about 2024-11-16T09:58:39,135 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731751108989.7c7f87a6124bfe2f4049ee7803e0f98c., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 , the old value is region=TestAcidGuarantees,,1731751108989.7c7f87a6124bfe2f4049ee7803e0f98c., hostname=037c58a19ddb,37267,1731751077844, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c7f87a6124bfe2f4049ee7803e0f98c, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:58:39,135 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731751108989.7c7f87a6124bfe2f4049ee7803e0f98c., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c7f87a6124bfe2f4049ee7803e0f98c, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:58:39,135 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731751108989.7c7f87a6124bfe2f4049ee7803e0f98c., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 because the exception is null or not the one we care about 2024-11-16T09:58:39,135 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c7f87a6124bfe2f4049ee7803e0f98c, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T09:58:39,135 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52400 deadline: 1731751179134, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c7f87a6124bfe2f4049ee7803e0f98c, server=037c58a19ddb,37267,1731751077844 2024-11-16T09:58:39,135 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c7f87a6124bfe2f4049ee7803e0f98c, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T09:58:39,135 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52420 deadline: 1731751179134, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c7f87a6124bfe2f4049ee7803e0f98c, server=037c58a19ddb,37267,1731751077844 2024-11-16T09:58:39,135 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731751108989.7c7f87a6124bfe2f4049ee7803e0f98c., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 , the old value is region=TestAcidGuarantees,,1731751108989.7c7f87a6124bfe2f4049ee7803e0f98c., hostname=037c58a19ddb,37267,1731751077844, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c7f87a6124bfe2f4049ee7803e0f98c, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:58:39,136 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731751108989.7c7f87a6124bfe2f4049ee7803e0f98c., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c7f87a6124bfe2f4049ee7803e0f98c, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:58:39,136 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731751108989.7c7f87a6124bfe2f4049ee7803e0f98c., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 because the exception is null or not the one we care about 2024-11-16T09:58:39,136 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731751108989.7c7f87a6124bfe2f4049ee7803e0f98c., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 , the old value is region=TestAcidGuarantees,,1731751108989.7c7f87a6124bfe2f4049ee7803e0f98c., hostname=037c58a19ddb,37267,1731751077844, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c7f87a6124bfe2f4049ee7803e0f98c, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:58:39,136 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731751108989.7c7f87a6124bfe2f4049ee7803e0f98c., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c7f87a6124bfe2f4049ee7803e0f98c, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:58:39,136 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731751108989.7c7f87a6124bfe2f4049ee7803e0f98c., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 because the exception is null or not the one we care about 2024-11-16T09:58:39,153 DEBUG [RS:0;037c58a19ddb:37267-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/7c7f87a6124bfe2f4049ee7803e0f98c/.tmp/A/4ca189b9a842483db6e475afcc8899aa as hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/7c7f87a6124bfe2f4049ee7803e0f98c/A/4ca189b9a842483db6e475afcc8899aa 2024-11-16T09:58:39,159 INFO [RS:0;037c58a19ddb:37267-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 7c7f87a6124bfe2f4049ee7803e0f98c/A of 7c7f87a6124bfe2f4049ee7803e0f98c into 4ca189b9a842483db6e475afcc8899aa(size=30.5 K), total size for store is 30.5 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-16T09:58:39,159 DEBUG [RS:0;037c58a19ddb:37267-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 7c7f87a6124bfe2f4049ee7803e0f98c: 2024-11-16T09:58:39,160 INFO [RS:0;037c58a19ddb:37267-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731751108989.7c7f87a6124bfe2f4049ee7803e0f98c., storeName=7c7f87a6124bfe2f4049ee7803e0f98c/A, priority=13, startTime=1731751118650; duration=0sec 2024-11-16T09:58:39,160 DEBUG [RS:0;037c58a19ddb:37267-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-16T09:58:39,160 DEBUG [RS:0;037c58a19ddb:37267-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 7c7f87a6124bfe2f4049ee7803e0f98c:A 2024-11-16T09:58:39,242 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T09:58:39,247 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241116e32f3ab2cc8743e18287927778b733e6_7c7f87a6124bfe2f4049ee7803e0f98c to hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241116e32f3ab2cc8743e18287927778b733e6_7c7f87a6124bfe2f4049ee7803e0f98c 2024-11-16T09:58:39,248 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/7c7f87a6124bfe2f4049ee7803e0f98c/.tmp/A/64fcc2a7d41e439a952ea7e0969d6e41, store: [table=TestAcidGuarantees family=A region=7c7f87a6124bfe2f4049ee7803e0f98c] 2024-11-16T09:58:39,249 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/7c7f87a6124bfe2f4049ee7803e0f98c/.tmp/A/64fcc2a7d41e439a952ea7e0969d6e41 is 175, key is test_row_0/A:col10/1731751118154/Put/seqid=0 2024-11-16T09:58:39,258 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37591 is added to blk_1073742001_1177 (size=31005) 2024-11-16T09:58:39,258 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=37267 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=43 2024-11-16T09:58:39,258 DEBUG [RS_FLUSH_OPERATIONS-regionserver/037c58a19ddb:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731751108989.7c7f87a6124bfe2f4049ee7803e0f98c. 2024-11-16T09:58:39,258 DEBUG [RS_FLUSH_OPERATIONS-regionserver/037c58a19ddb:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731751108989.7c7f87a6124bfe2f4049ee7803e0f98c. as already flushing 2024-11-16T09:58:39,259 DEBUG [RS_FLUSH_OPERATIONS-regionserver/037c58a19ddb:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731751108989.7c7f87a6124bfe2f4049ee7803e0f98c. 2024-11-16T09:58:39,259 ERROR [RS_FLUSH_OPERATIONS-regionserver/037c58a19ddb:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] handler.RSProcedureHandler(58): pid=43 java.io.IOException: Unable to complete flush {ENCODED => 7c7f87a6124bfe2f4049ee7803e0f98c, NAME => 'TestAcidGuarantees,,1731751108989.7c7f87a6124bfe2f4049ee7803e0f98c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T09:58:39,259 DEBUG [RS_FLUSH_OPERATIONS-regionserver/037c58a19ddb:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=43 java.io.IOException: Unable to complete flush {ENCODED => 7c7f87a6124bfe2f4049ee7803e0f98c, NAME => 'TestAcidGuarantees,,1731751108989.7c7f87a6124bfe2f4049ee7803e0f98c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T09:58:39,260 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37877 {}] master.HMaster(4177): Remote procedure failed, pid=43 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 7c7f87a6124bfe2f4049ee7803e0f98c, NAME => 'TestAcidGuarantees,,1731751108989.7c7f87a6124bfe2f4049ee7803e0f98c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 7c7f87a6124bfe2f4049ee7803e0f98c, NAME => 'TestAcidGuarantees,,1731751108989.7c7f87a6124bfe2f4049ee7803e0f98c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T09:58:39,260 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=131, memsize=53.7 K, hasBloomFilter=true, into tmp file hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/7c7f87a6124bfe2f4049ee7803e0f98c/.tmp/A/64fcc2a7d41e439a952ea7e0969d6e41 2024-11-16T09:58:39,270 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/7c7f87a6124bfe2f4049ee7803e0f98c/.tmp/B/ed2ff962b6c54b2e95ec44bbec83832f is 50, key is test_row_0/B:col10/1731751118154/Put/seqid=0 2024-11-16T09:58:39,281 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37591 is added to blk_1073742002_1178 (size=12051) 2024-11-16T09:58:39,313 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c7f87a6124bfe2f4049ee7803e0f98c, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T09:58:39,313 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52428 deadline: 1731751179312, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c7f87a6124bfe2f4049ee7803e0f98c, server=037c58a19ddb,37267,1731751077844 2024-11-16T09:58:39,313 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731751108989.7c7f87a6124bfe2f4049ee7803e0f98c., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 , the old value is region=TestAcidGuarantees,,1731751108989.7c7f87a6124bfe2f4049ee7803e0f98c., hostname=037c58a19ddb,37267,1731751077844, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c7f87a6124bfe2f4049ee7803e0f98c, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:58:39,313 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731751108989.7c7f87a6124bfe2f4049ee7803e0f98c., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c7f87a6124bfe2f4049ee7803e0f98c, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:58:39,313 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731751108989.7c7f87a6124bfe2f4049ee7803e0f98c., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 because the exception is null or not the one we care about 2024-11-16T09:58:39,412 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=37267 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=43 2024-11-16T09:58:39,413 DEBUG [RS_FLUSH_OPERATIONS-regionserver/037c58a19ddb:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731751108989.7c7f87a6124bfe2f4049ee7803e0f98c. 2024-11-16T09:58:39,413 DEBUG [RS_FLUSH_OPERATIONS-regionserver/037c58a19ddb:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731751108989.7c7f87a6124bfe2f4049ee7803e0f98c. as already flushing 2024-11-16T09:58:39,413 DEBUG [RS_FLUSH_OPERATIONS-regionserver/037c58a19ddb:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731751108989.7c7f87a6124bfe2f4049ee7803e0f98c. 2024-11-16T09:58:39,413 ERROR [RS_FLUSH_OPERATIONS-regionserver/037c58a19ddb:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] handler.RSProcedureHandler(58): pid=43 java.io.IOException: Unable to complete flush {ENCODED => 7c7f87a6124bfe2f4049ee7803e0f98c, NAME => 'TestAcidGuarantees,,1731751108989.7c7f87a6124bfe2f4049ee7803e0f98c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T09:58:39,413 DEBUG [RS_FLUSH_OPERATIONS-regionserver/037c58a19ddb:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=43 java.io.IOException: Unable to complete flush {ENCODED => 7c7f87a6124bfe2f4049ee7803e0f98c, NAME => 'TestAcidGuarantees,,1731751108989.7c7f87a6124bfe2f4049ee7803e0f98c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T09:58:39,414 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37877 {}] master.HMaster(4177): Remote procedure failed, pid=43 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 7c7f87a6124bfe2f4049ee7803e0f98c, NAME => 'TestAcidGuarantees,,1731751108989.7c7f87a6124bfe2f4049ee7803e0f98c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 7c7f87a6124bfe2f4049ee7803e0f98c, NAME => 'TestAcidGuarantees,,1731751108989.7c7f87a6124bfe2f4049ee7803e0f98c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T09:58:39,443 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c7f87a6124bfe2f4049ee7803e0f98c, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T09:58:39,444 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52396 deadline: 1731751179442, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c7f87a6124bfe2f4049ee7803e0f98c, server=037c58a19ddb,37267,1731751077844 2024-11-16T09:58:39,443 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c7f87a6124bfe2f4049ee7803e0f98c, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T09:58:39,444 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c7f87a6124bfe2f4049ee7803e0f98c, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T09:58:39,444 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731751108989.7c7f87a6124bfe2f4049ee7803e0f98c., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 , the old value is region=TestAcidGuarantees,,1731751108989.7c7f87a6124bfe2f4049ee7803e0f98c., hostname=037c58a19ddb,37267,1731751077844, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c7f87a6124bfe2f4049ee7803e0f98c, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:58:39,444 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731751108989.7c7f87a6124bfe2f4049ee7803e0f98c., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c7f87a6124bfe2f4049ee7803e0f98c, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:58:39,444 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731751108989.7c7f87a6124bfe2f4049ee7803e0f98c., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 because the exception is null or not the one we care about 2024-11-16T09:58:39,444 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52400 deadline: 1731751179443, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c7f87a6124bfe2f4049ee7803e0f98c, server=037c58a19ddb,37267,1731751077844 2024-11-16T09:58:39,444 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52420 deadline: 1731751179443, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c7f87a6124bfe2f4049ee7803e0f98c, server=037c58a19ddb,37267,1731751077844 2024-11-16T09:58:39,444 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731751108989.7c7f87a6124bfe2f4049ee7803e0f98c., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 , the old value is region=TestAcidGuarantees,,1731751108989.7c7f87a6124bfe2f4049ee7803e0f98c., hostname=037c58a19ddb,37267,1731751077844, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c7f87a6124bfe2f4049ee7803e0f98c, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:58:39,444 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731751108989.7c7f87a6124bfe2f4049ee7803e0f98c., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 , the old value is region=TestAcidGuarantees,,1731751108989.7c7f87a6124bfe2f4049ee7803e0f98c., hostname=037c58a19ddb,37267,1731751077844, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c7f87a6124bfe2f4049ee7803e0f98c, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:58:39,444 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731751108989.7c7f87a6124bfe2f4049ee7803e0f98c., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c7f87a6124bfe2f4049ee7803e0f98c, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:58:39,444 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731751108989.7c7f87a6124bfe2f4049ee7803e0f98c., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c7f87a6124bfe2f4049ee7803e0f98c, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:58:39,444 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731751108989.7c7f87a6124bfe2f4049ee7803e0f98c., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 because the exception is null or not the one we care about 2024-11-16T09:58:39,445 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731751108989.7c7f87a6124bfe2f4049ee7803e0f98c., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 because the exception is null or not the one we care about 2024-11-16T09:58:39,445 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c7f87a6124bfe2f4049ee7803e0f98c, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T09:58:39,445 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52392 deadline: 1731751179444, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c7f87a6124bfe2f4049ee7803e0f98c, server=037c58a19ddb,37267,1731751077844 2024-11-16T09:58:39,447 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731751108989.7c7f87a6124bfe2f4049ee7803e0f98c., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 , the old value is region=TestAcidGuarantees,,1731751108989.7c7f87a6124bfe2f4049ee7803e0f98c., hostname=037c58a19ddb,37267,1731751077844, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c7f87a6124bfe2f4049ee7803e0f98c, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:58:39,447 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731751108989.7c7f87a6124bfe2f4049ee7803e0f98c., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c7f87a6124bfe2f4049ee7803e0f98c, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:58:39,447 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731751108989.7c7f87a6124bfe2f4049ee7803e0f98c., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 because the exception is null or not the one we care about 2024-11-16T09:58:39,567 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=37267 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=43 2024-11-16T09:58:39,568 DEBUG [RS_FLUSH_OPERATIONS-regionserver/037c58a19ddb:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731751108989.7c7f87a6124bfe2f4049ee7803e0f98c. 2024-11-16T09:58:39,568 DEBUG [RS_FLUSH_OPERATIONS-regionserver/037c58a19ddb:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731751108989.7c7f87a6124bfe2f4049ee7803e0f98c. as already flushing 2024-11-16T09:58:39,568 DEBUG [RS_FLUSH_OPERATIONS-regionserver/037c58a19ddb:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731751108989.7c7f87a6124bfe2f4049ee7803e0f98c. 2024-11-16T09:58:39,568 ERROR [RS_FLUSH_OPERATIONS-regionserver/037c58a19ddb:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] handler.RSProcedureHandler(58): pid=43 java.io.IOException: Unable to complete flush {ENCODED => 7c7f87a6124bfe2f4049ee7803e0f98c, NAME => 'TestAcidGuarantees,,1731751108989.7c7f87a6124bfe2f4049ee7803e0f98c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T09:58:39,569 DEBUG [RS_FLUSH_OPERATIONS-regionserver/037c58a19ddb:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=43 java.io.IOException: Unable to complete flush {ENCODED => 7c7f87a6124bfe2f4049ee7803e0f98c, NAME => 'TestAcidGuarantees,,1731751108989.7c7f87a6124bfe2f4049ee7803e0f98c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T09:58:39,569 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37877 {}] master.HMaster(4177): Remote procedure failed, pid=43 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 7c7f87a6124bfe2f4049ee7803e0f98c, NAME => 'TestAcidGuarantees,,1731751108989.7c7f87a6124bfe2f4049ee7803e0f98c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 7c7f87a6124bfe2f4049ee7803e0f98c, NAME => 'TestAcidGuarantees,,1731751108989.7c7f87a6124bfe2f4049ee7803e0f98c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T09:58:39,623 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37877 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=42 2024-11-16T09:58:39,683 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=53.67 KB at sequenceid=131 (bloomFilter=true), to=hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/7c7f87a6124bfe2f4049ee7803e0f98c/.tmp/B/ed2ff962b6c54b2e95ec44bbec83832f 2024-11-16T09:58:39,693 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/7c7f87a6124bfe2f4049ee7803e0f98c/.tmp/C/3ec8bedcb3c04f9f96a9d4d23b6bd596 is 50, key is test_row_0/C:col10/1731751118154/Put/seqid=0 2024-11-16T09:58:39,705 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37591 is added to blk_1073742003_1179 (size=12051) 2024-11-16T09:58:39,706 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=53.67 KB at sequenceid=131 (bloomFilter=true), to=hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/7c7f87a6124bfe2f4049ee7803e0f98c/.tmp/C/3ec8bedcb3c04f9f96a9d4d23b6bd596 2024-11-16T09:58:39,720 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/7c7f87a6124bfe2f4049ee7803e0f98c/.tmp/A/64fcc2a7d41e439a952ea7e0969d6e41 as hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/7c7f87a6124bfe2f4049ee7803e0f98c/A/64fcc2a7d41e439a952ea7e0969d6e41 2024-11-16T09:58:39,722 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=37267 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=43 2024-11-16T09:58:39,723 DEBUG [RS_FLUSH_OPERATIONS-regionserver/037c58a19ddb:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731751108989.7c7f87a6124bfe2f4049ee7803e0f98c. 2024-11-16T09:58:39,723 DEBUG [RS_FLUSH_OPERATIONS-regionserver/037c58a19ddb:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731751108989.7c7f87a6124bfe2f4049ee7803e0f98c. as already flushing 2024-11-16T09:58:39,723 DEBUG [RS_FLUSH_OPERATIONS-regionserver/037c58a19ddb:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731751108989.7c7f87a6124bfe2f4049ee7803e0f98c. 2024-11-16T09:58:39,723 ERROR [RS_FLUSH_OPERATIONS-regionserver/037c58a19ddb:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] handler.RSProcedureHandler(58): pid=43 java.io.IOException: Unable to complete flush {ENCODED => 7c7f87a6124bfe2f4049ee7803e0f98c, NAME => 'TestAcidGuarantees,,1731751108989.7c7f87a6124bfe2f4049ee7803e0f98c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T09:58:39,723 DEBUG [RS_FLUSH_OPERATIONS-regionserver/037c58a19ddb:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=43 java.io.IOException: Unable to complete flush {ENCODED => 7c7f87a6124bfe2f4049ee7803e0f98c, NAME => 'TestAcidGuarantees,,1731751108989.7c7f87a6124bfe2f4049ee7803e0f98c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T09:58:39,724 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37877 {}] master.HMaster(4177): Remote procedure failed, pid=43 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 7c7f87a6124bfe2f4049ee7803e0f98c, NAME => 'TestAcidGuarantees,,1731751108989.7c7f87a6124bfe2f4049ee7803e0f98c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 7c7f87a6124bfe2f4049ee7803e0f98c, NAME => 'TestAcidGuarantees,,1731751108989.7c7f87a6124bfe2f4049ee7803e0f98c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T09:58:39,726 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/7c7f87a6124bfe2f4049ee7803e0f98c/A/64fcc2a7d41e439a952ea7e0969d6e41, entries=150, sequenceid=131, filesize=30.3 K 2024-11-16T09:58:39,728 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/7c7f87a6124bfe2f4049ee7803e0f98c/.tmp/B/ed2ff962b6c54b2e95ec44bbec83832f as hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/7c7f87a6124bfe2f4049ee7803e0f98c/B/ed2ff962b6c54b2e95ec44bbec83832f 2024-11-16T09:58:39,736 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/7c7f87a6124bfe2f4049ee7803e0f98c/B/ed2ff962b6c54b2e95ec44bbec83832f, entries=150, sequenceid=131, filesize=11.8 K 2024-11-16T09:58:39,737 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/7c7f87a6124bfe2f4049ee7803e0f98c/.tmp/C/3ec8bedcb3c04f9f96a9d4d23b6bd596 as hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/7c7f87a6124bfe2f4049ee7803e0f98c/C/3ec8bedcb3c04f9f96a9d4d23b6bd596 2024-11-16T09:58:39,744 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/7c7f87a6124bfe2f4049ee7803e0f98c/C/3ec8bedcb3c04f9f96a9d4d23b6bd596, entries=150, sequenceid=131, filesize=11.8 K 2024-11-16T09:58:39,745 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~161.02 KB/164880, heapSize ~422.58 KB/432720, currentSize=46.96 KB/48090 for 7c7f87a6124bfe2f4049ee7803e0f98c in 957ms, sequenceid=131, compaction requested=true 2024-11-16T09:58:39,745 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 7c7f87a6124bfe2f4049ee7803e0f98c: 2024-11-16T09:58:39,745 DEBUG [RS:0;037c58a19ddb:37267-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 2 store files, 0 compacting, 2 eligible, 16 blocking 2024-11-16T09:58:39,745 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 7c7f87a6124bfe2f4049ee7803e0f98c:A, priority=-2147483648, current under compaction store size is 1 2024-11-16T09:58:39,745 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-16T09:58:39,745 DEBUG [RS:0;037c58a19ddb:37267-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 2 store files, 0 compacting, 2 eligible, 16 blocking 2024-11-16T09:58:39,746 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 7c7f87a6124bfe2f4049ee7803e0f98c:B, priority=-2147483648, current under compaction store size is 2 2024-11-16T09:58:39,746 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-16T09:58:39,746 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 7c7f87a6124bfe2f4049ee7803e0f98c:C, priority=-2147483648, current under compaction store size is 3 2024-11-16T09:58:39,746 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-16T09:58:39,746 DEBUG [RS:0;037c58a19ddb:37267-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 0 files of size 0 starting at candidate #-1 after considering 0 permutations with 0 in ratio 2024-11-16T09:58:39,746 DEBUG [RS:0;037c58a19ddb:37267-shortCompactions-0 {}] compactions.SortedCompactionPolicy(232): Not compacting files because we only have 0 files ready for compaction. Need 3 to initiate. 2024-11-16T09:58:39,747 DEBUG [RS:0;037c58a19ddb:37267-shortCompactions-0 {}] regionserver.CompactSplit(450): Not compacting TestAcidGuarantees,,1731751108989.7c7f87a6124bfe2f4049ee7803e0f98c. because compaction request was cancelled 2024-11-16T09:58:39,747 DEBUG [RS:0;037c58a19ddb:37267-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 7c7f87a6124bfe2f4049ee7803e0f98c:A 2024-11-16T09:58:39,747 DEBUG [RS:0;037c58a19ddb:37267-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-16T09:58:39,747 DEBUG [RS:0;037c58a19ddb:37267-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 0 files of size 0 starting at candidate #-1 after considering 0 permutations with 0 in ratio 2024-11-16T09:58:39,747 DEBUG [RS:0;037c58a19ddb:37267-longCompactions-0 {}] compactions.SortedCompactionPolicy(232): Not compacting files because we only have 0 files ready for compaction. Need 3 to initiate. 2024-11-16T09:58:39,747 DEBUG [RS:0;037c58a19ddb:37267-longCompactions-0 {}] regionserver.CompactSplit(450): Not compacting TestAcidGuarantees,,1731751108989.7c7f87a6124bfe2f4049ee7803e0f98c. because compaction request was cancelled 2024-11-16T09:58:39,747 DEBUG [RS:0;037c58a19ddb:37267-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 7c7f87a6124bfe2f4049ee7803e0f98c:B 2024-11-16T09:58:39,748 DEBUG [RS:0;037c58a19ddb:37267-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36225 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-16T09:58:39,748 DEBUG [RS:0;037c58a19ddb:37267-shortCompactions-0 {}] regionserver.HStore(1541): 7c7f87a6124bfe2f4049ee7803e0f98c/C is initiating minor compaction (all files) 2024-11-16T09:58:39,748 INFO [RS:0;037c58a19ddb:37267-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 7c7f87a6124bfe2f4049ee7803e0f98c/C in TestAcidGuarantees,,1731751108989.7c7f87a6124bfe2f4049ee7803e0f98c. 2024-11-16T09:58:39,748 INFO [RS:0;037c58a19ddb:37267-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/7c7f87a6124bfe2f4049ee7803e0f98c/C/f7adaf3740b84fe18c83f162e49403a7, hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/7c7f87a6124bfe2f4049ee7803e0f98c/C/821acc26a10c434aad651ccdb01cfe0a, hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/7c7f87a6124bfe2f4049ee7803e0f98c/C/3ec8bedcb3c04f9f96a9d4d23b6bd596] into tmpdir=hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/7c7f87a6124bfe2f4049ee7803e0f98c/.tmp, totalSize=35.4 K 2024-11-16T09:58:39,749 DEBUG [RS:0;037c58a19ddb:37267-shortCompactions-0 {}] compactions.Compactor(225): Compacting f7adaf3740b84fe18c83f162e49403a7, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=90, earliestPutTs=1731751115756 2024-11-16T09:58:39,750 DEBUG [RS:0;037c58a19ddb:37267-shortCompactions-0 {}] compactions.Compactor(225): Compacting 821acc26a10c434aad651ccdb01cfe0a, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=103, earliestPutTs=1731751117995 2024-11-16T09:58:39,751 DEBUG [RS:0;037c58a19ddb:37267-shortCompactions-0 {}] compactions.Compactor(225): Compacting 3ec8bedcb3c04f9f96a9d4d23b6bd596, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=131, earliestPutTs=1731751118146 2024-11-16T09:58:39,759 INFO [RS:0;037c58a19ddb:37267-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 7c7f87a6124bfe2f4049ee7803e0f98c#C#compaction#155 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-16T09:58:39,760 DEBUG [RS:0;037c58a19ddb:37267-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/7c7f87a6124bfe2f4049ee7803e0f98c/.tmp/C/2e3828d7d79b488a9c4d75f3a4e6af2d is 50, key is test_row_0/C:col10/1731751118154/Put/seqid=0 2024-11-16T09:58:39,797 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37591 is added to blk_1073742004_1180 (size=12325) 2024-11-16T09:58:39,876 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=37267 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=43 2024-11-16T09:58:39,877 DEBUG [RS_FLUSH_OPERATIONS-regionserver/037c58a19ddb:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731751108989.7c7f87a6124bfe2f4049ee7803e0f98c. 2024-11-16T09:58:39,877 INFO [RS_FLUSH_OPERATIONS-regionserver/037c58a19ddb:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegion(2902): Flushing 7c7f87a6124bfe2f4049ee7803e0f98c 3/3 column families, dataSize=46.96 KB heapSize=123.80 KB 2024-11-16T09:58:39,888 DEBUG [RS_FLUSH_OPERATIONS-regionserver/037c58a19ddb:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411165a2d53f04455440790a250a3b092bca6_7c7f87a6124bfe2f4049ee7803e0f98c is 50, key is test_row_0/A:col10/1731751118807/Put/seqid=0 2024-11-16T09:58:39,906 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37591 is added to blk_1073742005_1181 (size=12304) 2024-11-16T09:58:39,962 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] regionserver.HRegion(8855): Flush requested on 7c7f87a6124bfe2f4049ee7803e0f98c 2024-11-16T09:58:39,962 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731751108989.7c7f87a6124bfe2f4049ee7803e0f98c. as already flushing 2024-11-16T09:58:40,095 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c7f87a6124bfe2f4049ee7803e0f98c, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T09:58:40,096 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52392 deadline: 1731751180089, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c7f87a6124bfe2f4049ee7803e0f98c, server=037c58a19ddb,37267,1731751077844 2024-11-16T09:58:40,096 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731751108989.7c7f87a6124bfe2f4049ee7803e0f98c., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 , the old value is region=TestAcidGuarantees,,1731751108989.7c7f87a6124bfe2f4049ee7803e0f98c., hostname=037c58a19ddb,37267,1731751077844, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c7f87a6124bfe2f4049ee7803e0f98c, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:58:40,096 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731751108989.7c7f87a6124bfe2f4049ee7803e0f98c., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c7f87a6124bfe2f4049ee7803e0f98c, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:58:40,096 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731751108989.7c7f87a6124bfe2f4049ee7803e0f98c., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 because the exception is null or not the one we care about 2024-11-16T09:58:40,091 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c7f87a6124bfe2f4049ee7803e0f98c, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T09:58:40,100 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52400 deadline: 1731751180077, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c7f87a6124bfe2f4049ee7803e0f98c, server=037c58a19ddb,37267,1731751077844 2024-11-16T09:58:40,100 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731751108989.7c7f87a6124bfe2f4049ee7803e0f98c., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 , the old value is region=TestAcidGuarantees,,1731751108989.7c7f87a6124bfe2f4049ee7803e0f98c., hostname=037c58a19ddb,37267,1731751077844, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c7f87a6124bfe2f4049ee7803e0f98c, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:58:40,100 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c7f87a6124bfe2f4049ee7803e0f98c, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T09:58:40,100 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731751108989.7c7f87a6124bfe2f4049ee7803e0f98c., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c7f87a6124bfe2f4049ee7803e0f98c, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:58:40,100 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731751108989.7c7f87a6124bfe2f4049ee7803e0f98c., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 because the exception is null or not the one we care about 2024-11-16T09:58:40,101 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52396 deadline: 1731751180092, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c7f87a6124bfe2f4049ee7803e0f98c, server=037c58a19ddb,37267,1731751077844 2024-11-16T09:58:40,101 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c7f87a6124bfe2f4049ee7803e0f98c, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T09:58:40,101 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52420 deadline: 1731751180092, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c7f87a6124bfe2f4049ee7803e0f98c, server=037c58a19ddb,37267,1731751077844 2024-11-16T09:58:40,102 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731751108989.7c7f87a6124bfe2f4049ee7803e0f98c., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 , the old value is region=TestAcidGuarantees,,1731751108989.7c7f87a6124bfe2f4049ee7803e0f98c., hostname=037c58a19ddb,37267,1731751077844, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c7f87a6124bfe2f4049ee7803e0f98c, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:58:40,102 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731751108989.7c7f87a6124bfe2f4049ee7803e0f98c., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c7f87a6124bfe2f4049ee7803e0f98c, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:58:40,102 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731751108989.7c7f87a6124bfe2f4049ee7803e0f98c., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 because the exception is null or not the one we care about 2024-11-16T09:58:40,102 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731751108989.7c7f87a6124bfe2f4049ee7803e0f98c., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 , the old value is region=TestAcidGuarantees,,1731751108989.7c7f87a6124bfe2f4049ee7803e0f98c., hostname=037c58a19ddb,37267,1731751077844, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c7f87a6124bfe2f4049ee7803e0f98c, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:58:40,102 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731751108989.7c7f87a6124bfe2f4049ee7803e0f98c., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c7f87a6124bfe2f4049ee7803e0f98c, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:58:40,102 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731751108989.7c7f87a6124bfe2f4049ee7803e0f98c., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 because the exception is null or not the one we care about 2024-11-16T09:58:40,205 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c7f87a6124bfe2f4049ee7803e0f98c, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T09:58:40,206 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52392 deadline: 1731751180204, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c7f87a6124bfe2f4049ee7803e0f98c, server=037c58a19ddb,37267,1731751077844 2024-11-16T09:58:40,206 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731751108989.7c7f87a6124bfe2f4049ee7803e0f98c., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 , the old value is region=TestAcidGuarantees,,1731751108989.7c7f87a6124bfe2f4049ee7803e0f98c., hostname=037c58a19ddb,37267,1731751077844, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c7f87a6124bfe2f4049ee7803e0f98c, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:58:40,206 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731751108989.7c7f87a6124bfe2f4049ee7803e0f98c., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c7f87a6124bfe2f4049ee7803e0f98c, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:58:40,206 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731751108989.7c7f87a6124bfe2f4049ee7803e0f98c., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 because the exception is null or not the one we care about 2024-11-16T09:58:40,206 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c7f87a6124bfe2f4049ee7803e0f98c, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T09:58:40,207 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52400 deadline: 1731751180205, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c7f87a6124bfe2f4049ee7803e0f98c, server=037c58a19ddb,37267,1731751077844 2024-11-16T09:58:40,207 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731751108989.7c7f87a6124bfe2f4049ee7803e0f98c., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 , the old value is region=TestAcidGuarantees,,1731751108989.7c7f87a6124bfe2f4049ee7803e0f98c., hostname=037c58a19ddb,37267,1731751077844, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c7f87a6124bfe2f4049ee7803e0f98c, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:58:40,207 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731751108989.7c7f87a6124bfe2f4049ee7803e0f98c., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c7f87a6124bfe2f4049ee7803e0f98c, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:58:40,207 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731751108989.7c7f87a6124bfe2f4049ee7803e0f98c., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 because the exception is null or not the one we care about 2024-11-16T09:58:40,210 DEBUG [RS:0;037c58a19ddb:37267-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/7c7f87a6124bfe2f4049ee7803e0f98c/.tmp/C/2e3828d7d79b488a9c4d75f3a4e6af2d as hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/7c7f87a6124bfe2f4049ee7803e0f98c/C/2e3828d7d79b488a9c4d75f3a4e6af2d 2024-11-16T09:58:40,219 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c7f87a6124bfe2f4049ee7803e0f98c, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T09:58:40,219 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52420 deadline: 1731751180213, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c7f87a6124bfe2f4049ee7803e0f98c, server=037c58a19ddb,37267,1731751077844 2024-11-16T09:58:40,219 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c7f87a6124bfe2f4049ee7803e0f98c, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T09:58:40,219 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52396 deadline: 1731751180213, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c7f87a6124bfe2f4049ee7803e0f98c, server=037c58a19ddb,37267,1731751077844 2024-11-16T09:58:40,220 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731751108989.7c7f87a6124bfe2f4049ee7803e0f98c., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 , the old value is region=TestAcidGuarantees,,1731751108989.7c7f87a6124bfe2f4049ee7803e0f98c., hostname=037c58a19ddb,37267,1731751077844, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c7f87a6124bfe2f4049ee7803e0f98c, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:58:40,220 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731751108989.7c7f87a6124bfe2f4049ee7803e0f98c., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c7f87a6124bfe2f4049ee7803e0f98c, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:58:40,220 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731751108989.7c7f87a6124bfe2f4049ee7803e0f98c., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 because the exception is null or not the one we care about 2024-11-16T09:58:40,220 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731751108989.7c7f87a6124bfe2f4049ee7803e0f98c., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 , the old value is region=TestAcidGuarantees,,1731751108989.7c7f87a6124bfe2f4049ee7803e0f98c., hostname=037c58a19ddb,37267,1731751077844, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c7f87a6124bfe2f4049ee7803e0f98c, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:58:40,220 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731751108989.7c7f87a6124bfe2f4049ee7803e0f98c., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c7f87a6124bfe2f4049ee7803e0f98c, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:58:40,220 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731751108989.7c7f87a6124bfe2f4049ee7803e0f98c., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 because the exception is null or not the one we care about 2024-11-16T09:58:40,230 INFO [RS:0;037c58a19ddb:37267-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 7c7f87a6124bfe2f4049ee7803e0f98c/C of 7c7f87a6124bfe2f4049ee7803e0f98c into 2e3828d7d79b488a9c4d75f3a4e6af2d(size=12.0 K), total size for store is 12.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-16T09:58:40,230 DEBUG [RS:0;037c58a19ddb:37267-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 7c7f87a6124bfe2f4049ee7803e0f98c: 2024-11-16T09:58:40,230 INFO [RS:0;037c58a19ddb:37267-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731751108989.7c7f87a6124bfe2f4049ee7803e0f98c., storeName=7c7f87a6124bfe2f4049ee7803e0f98c/C, priority=13, startTime=1731751119746; duration=0sec 2024-11-16T09:58:40,230 DEBUG [RS:0;037c58a19ddb:37267-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-16T09:58:40,230 DEBUG [RS:0;037c58a19ddb:37267-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 7c7f87a6124bfe2f4049ee7803e0f98c:C 2024-11-16T09:58:40,307 DEBUG [RS_FLUSH_OPERATIONS-regionserver/037c58a19ddb:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T09:58:40,315 INFO [RS_FLUSH_OPERATIONS-regionserver/037c58a19ddb:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411165a2d53f04455440790a250a3b092bca6_7c7f87a6124bfe2f4049ee7803e0f98c to hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411165a2d53f04455440790a250a3b092bca6_7c7f87a6124bfe2f4049ee7803e0f98c 2024-11-16T09:58:40,316 DEBUG [RS_FLUSH_OPERATIONS-regionserver/037c58a19ddb:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/7c7f87a6124bfe2f4049ee7803e0f98c/.tmp/A/8c843a2fb4074619b6de587720d98901, store: [table=TestAcidGuarantees family=A region=7c7f87a6124bfe2f4049ee7803e0f98c] 2024-11-16T09:58:40,316 DEBUG [RS_FLUSH_OPERATIONS-regionserver/037c58a19ddb:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/7c7f87a6124bfe2f4049ee7803e0f98c/.tmp/A/8c843a2fb4074619b6de587720d98901 is 175, key is test_row_0/A:col10/1731751118807/Put/seqid=0 2024-11-16T09:58:40,327 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c7f87a6124bfe2f4049ee7803e0f98c, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T09:58:40,327 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52428 deadline: 1731751180323, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c7f87a6124bfe2f4049ee7803e0f98c, server=037c58a19ddb,37267,1731751077844 2024-11-16T09:58:40,327 WARN [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRpcRetryingCaller(177): Call to 037c58a19ddb,37267,1731751077844 for 'test_row_1' in 7c7f87a6124bfe2f4049ee7803e0f98c of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2171 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c7f87a6124bfe2f4049ee7803e0f98c, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor44.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c7f87a6124bfe2f4049ee7803e0f98c, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-16T09:58:40,328 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731751108989.7c7f87a6124bfe2f4049ee7803e0f98c., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 , the old value is region=TestAcidGuarantees,,1731751108989.7c7f87a6124bfe2f4049ee7803e0f98c., hostname=037c58a19ddb,37267,1731751077844, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c7f87a6124bfe2f4049ee7803e0f98c, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:58:40,328 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731751108989.7c7f87a6124bfe2f4049ee7803e0f98c., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c7f87a6124bfe2f4049ee7803e0f98c, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:58:40,328 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731751108989.7c7f87a6124bfe2f4049ee7803e0f98c., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 because the exception is null or not the one we care about 2024-11-16T09:58:40,343 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37591 is added to blk_1073742006_1182 (size=31105) 2024-11-16T09:58:40,413 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c7f87a6124bfe2f4049ee7803e0f98c, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T09:58:40,414 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52392 deadline: 1731751180413, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c7f87a6124bfe2f4049ee7803e0f98c, server=037c58a19ddb,37267,1731751077844 2024-11-16T09:58:40,414 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c7f87a6124bfe2f4049ee7803e0f98c, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T09:58:40,414 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52400 deadline: 1731751180413, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c7f87a6124bfe2f4049ee7803e0f98c, server=037c58a19ddb,37267,1731751077844 2024-11-16T09:58:40,414 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731751108989.7c7f87a6124bfe2f4049ee7803e0f98c., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 , the old value is region=TestAcidGuarantees,,1731751108989.7c7f87a6124bfe2f4049ee7803e0f98c., hostname=037c58a19ddb,37267,1731751077844, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c7f87a6124bfe2f4049ee7803e0f98c, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:58:40,414 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731751108989.7c7f87a6124bfe2f4049ee7803e0f98c., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c7f87a6124bfe2f4049ee7803e0f98c, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:58:40,415 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731751108989.7c7f87a6124bfe2f4049ee7803e0f98c., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 because the exception is null or not the one we care about 2024-11-16T09:58:40,416 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731751108989.7c7f87a6124bfe2f4049ee7803e0f98c., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 , the old value is region=TestAcidGuarantees,,1731751108989.7c7f87a6124bfe2f4049ee7803e0f98c., hostname=037c58a19ddb,37267,1731751077844, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c7f87a6124bfe2f4049ee7803e0f98c, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:58:40,416 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731751108989.7c7f87a6124bfe2f4049ee7803e0f98c., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c7f87a6124bfe2f4049ee7803e0f98c, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:58:40,416 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731751108989.7c7f87a6124bfe2f4049ee7803e0f98c., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 because the exception is null or not the one we care about 2024-11-16T09:58:40,426 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c7f87a6124bfe2f4049ee7803e0f98c, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T09:58:40,427 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52396 deadline: 1731751180423, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c7f87a6124bfe2f4049ee7803e0f98c, server=037c58a19ddb,37267,1731751077844 2024-11-16T09:58:40,427 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731751108989.7c7f87a6124bfe2f4049ee7803e0f98c., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 , the old value is region=TestAcidGuarantees,,1731751108989.7c7f87a6124bfe2f4049ee7803e0f98c., hostname=037c58a19ddb,37267,1731751077844, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c7f87a6124bfe2f4049ee7803e0f98c, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:58:40,427 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c7f87a6124bfe2f4049ee7803e0f98c, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T09:58:40,427 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731751108989.7c7f87a6124bfe2f4049ee7803e0f98c., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c7f87a6124bfe2f4049ee7803e0f98c, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:58:40,427 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731751108989.7c7f87a6124bfe2f4049ee7803e0f98c., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 because the exception is null or not the one we care about 2024-11-16T09:58:40,427 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52420 deadline: 1731751180424, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c7f87a6124bfe2f4049ee7803e0f98c, server=037c58a19ddb,37267,1731751077844 2024-11-16T09:58:40,427 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731751108989.7c7f87a6124bfe2f4049ee7803e0f98c., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 , the old value is region=TestAcidGuarantees,,1731751108989.7c7f87a6124bfe2f4049ee7803e0f98c., hostname=037c58a19ddb,37267,1731751077844, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c7f87a6124bfe2f4049ee7803e0f98c, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:58:40,427 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731751108989.7c7f87a6124bfe2f4049ee7803e0f98c., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c7f87a6124bfe2f4049ee7803e0f98c, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:58:40,427 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731751108989.7c7f87a6124bfe2f4049ee7803e0f98c., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 because the exception is null or not the one we care about 2024-11-16T09:58:40,633 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37877 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=42 2024-11-16T09:58:40,726 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c7f87a6124bfe2f4049ee7803e0f98c, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T09:58:40,726 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52400 deadline: 1731751180723, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c7f87a6124bfe2f4049ee7803e0f98c, server=037c58a19ddb,37267,1731751077844 2024-11-16T09:58:40,726 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731751108989.7c7f87a6124bfe2f4049ee7803e0f98c., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 , the old value is region=TestAcidGuarantees,,1731751108989.7c7f87a6124bfe2f4049ee7803e0f98c., hostname=037c58a19ddb,37267,1731751077844, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c7f87a6124bfe2f4049ee7803e0f98c, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:58:40,726 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c7f87a6124bfe2f4049ee7803e0f98c, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T09:58:40,726 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731751108989.7c7f87a6124bfe2f4049ee7803e0f98c., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c7f87a6124bfe2f4049ee7803e0f98c, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:58:40,726 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731751108989.7c7f87a6124bfe2f4049ee7803e0f98c., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 because the exception is null or not the one we care about 2024-11-16T09:58:40,726 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52392 deadline: 1731751180723, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c7f87a6124bfe2f4049ee7803e0f98c, server=037c58a19ddb,37267,1731751077844 2024-11-16T09:58:40,727 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731751108989.7c7f87a6124bfe2f4049ee7803e0f98c., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 , the old value is region=TestAcidGuarantees,,1731751108989.7c7f87a6124bfe2f4049ee7803e0f98c., hostname=037c58a19ddb,37267,1731751077844, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c7f87a6124bfe2f4049ee7803e0f98c, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:58:40,727 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731751108989.7c7f87a6124bfe2f4049ee7803e0f98c., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c7f87a6124bfe2f4049ee7803e0f98c, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:58:40,727 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731751108989.7c7f87a6124bfe2f4049ee7803e0f98c., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 because the exception is null or not the one we care about 2024-11-16T09:58:40,737 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c7f87a6124bfe2f4049ee7803e0f98c, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T09:58:40,737 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52396 deadline: 1731751180733, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c7f87a6124bfe2f4049ee7803e0f98c, server=037c58a19ddb,37267,1731751077844 2024-11-16T09:58:40,738 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731751108989.7c7f87a6124bfe2f4049ee7803e0f98c., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 , the old value is region=TestAcidGuarantees,,1731751108989.7c7f87a6124bfe2f4049ee7803e0f98c., hostname=037c58a19ddb,37267,1731751077844, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c7f87a6124bfe2f4049ee7803e0f98c, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:58:40,738 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c7f87a6124bfe2f4049ee7803e0f98c, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T09:58:40,738 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731751108989.7c7f87a6124bfe2f4049ee7803e0f98c., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c7f87a6124bfe2f4049ee7803e0f98c, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:58:40,738 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731751108989.7c7f87a6124bfe2f4049ee7803e0f98c., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 because the exception is null or not the one we care about 2024-11-16T09:58:40,738 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52420 deadline: 1731751180733, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c7f87a6124bfe2f4049ee7803e0f98c, server=037c58a19ddb,37267,1731751077844 2024-11-16T09:58:40,738 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731751108989.7c7f87a6124bfe2f4049ee7803e0f98c., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 , the old value is region=TestAcidGuarantees,,1731751108989.7c7f87a6124bfe2f4049ee7803e0f98c., hostname=037c58a19ddb,37267,1731751077844, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c7f87a6124bfe2f4049ee7803e0f98c, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:58:40,738 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731751108989.7c7f87a6124bfe2f4049ee7803e0f98c., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c7f87a6124bfe2f4049ee7803e0f98c, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:58:40,738 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731751108989.7c7f87a6124bfe2f4049ee7803e0f98c., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 because the exception is null or not the one we care about 2024-11-16T09:58:40,744 INFO [RS_FLUSH_OPERATIONS-regionserver/037c58a19ddb:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=143, memsize=15.7 K, hasBloomFilter=true, into tmp file hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/7c7f87a6124bfe2f4049ee7803e0f98c/.tmp/A/8c843a2fb4074619b6de587720d98901 2024-11-16T09:58:40,754 DEBUG [RS_FLUSH_OPERATIONS-regionserver/037c58a19ddb:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/7c7f87a6124bfe2f4049ee7803e0f98c/.tmp/B/df3050c6104a4f5a843ce0d720c0bd8c is 50, key is test_row_0/B:col10/1731751118807/Put/seqid=0 2024-11-16T09:58:40,773 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37591 is added to blk_1073742007_1183 (size=12151) 2024-11-16T09:58:40,775 INFO [RS_FLUSH_OPERATIONS-regionserver/037c58a19ddb:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=143 (bloomFilter=true), to=hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/7c7f87a6124bfe2f4049ee7803e0f98c/.tmp/B/df3050c6104a4f5a843ce0d720c0bd8c 2024-11-16T09:58:40,785 DEBUG [RS_FLUSH_OPERATIONS-regionserver/037c58a19ddb:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/7c7f87a6124bfe2f4049ee7803e0f98c/.tmp/C/cd3257160203416e884bed4bb651ac18 is 50, key is test_row_0/C:col10/1731751118807/Put/seqid=0 2024-11-16T09:58:40,790 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37591 is added to blk_1073742008_1184 (size=12151) 2024-11-16T09:58:40,790 INFO [RS_FLUSH_OPERATIONS-regionserver/037c58a19ddb:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=143 (bloomFilter=true), to=hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/7c7f87a6124bfe2f4049ee7803e0f98c/.tmp/C/cd3257160203416e884bed4bb651ac18 2024-11-16T09:58:40,797 DEBUG [RS_FLUSH_OPERATIONS-regionserver/037c58a19ddb:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/7c7f87a6124bfe2f4049ee7803e0f98c/.tmp/A/8c843a2fb4074619b6de587720d98901 as hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/7c7f87a6124bfe2f4049ee7803e0f98c/A/8c843a2fb4074619b6de587720d98901 2024-11-16T09:58:40,802 INFO [RS_FLUSH_OPERATIONS-regionserver/037c58a19ddb:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/7c7f87a6124bfe2f4049ee7803e0f98c/A/8c843a2fb4074619b6de587720d98901, entries=150, sequenceid=143, filesize=30.4 K 2024-11-16T09:58:40,803 DEBUG [RS_FLUSH_OPERATIONS-regionserver/037c58a19ddb:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/7c7f87a6124bfe2f4049ee7803e0f98c/.tmp/B/df3050c6104a4f5a843ce0d720c0bd8c as hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/7c7f87a6124bfe2f4049ee7803e0f98c/B/df3050c6104a4f5a843ce0d720c0bd8c 2024-11-16T09:58:40,809 INFO [RS_FLUSH_OPERATIONS-regionserver/037c58a19ddb:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/7c7f87a6124bfe2f4049ee7803e0f98c/B/df3050c6104a4f5a843ce0d720c0bd8c, entries=150, sequenceid=143, filesize=11.9 K 2024-11-16T09:58:40,810 DEBUG [RS_FLUSH_OPERATIONS-regionserver/037c58a19ddb:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/7c7f87a6124bfe2f4049ee7803e0f98c/.tmp/C/cd3257160203416e884bed4bb651ac18 as hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/7c7f87a6124bfe2f4049ee7803e0f98c/C/cd3257160203416e884bed4bb651ac18 2024-11-16T09:58:40,819 INFO [RS_FLUSH_OPERATIONS-regionserver/037c58a19ddb:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/7c7f87a6124bfe2f4049ee7803e0f98c/C/cd3257160203416e884bed4bb651ac18, entries=150, sequenceid=143, filesize=11.9 K 2024-11-16T09:58:40,820 INFO [RS_FLUSH_OPERATIONS-regionserver/037c58a19ddb:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegion(3140): Finished flush of dataSize ~46.96 KB/48090, heapSize ~123.75 KB/126720, currentSize=161.02 KB/164880 for 7c7f87a6124bfe2f4049ee7803e0f98c in 943ms, sequenceid=143, compaction requested=true 2024-11-16T09:58:40,820 DEBUG [RS_FLUSH_OPERATIONS-regionserver/037c58a19ddb:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegion(2603): Flush status journal for 7c7f87a6124bfe2f4049ee7803e0f98c: 2024-11-16T09:58:40,820 DEBUG [RS_FLUSH_OPERATIONS-regionserver/037c58a19ddb:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731751108989.7c7f87a6124bfe2f4049ee7803e0f98c. 2024-11-16T09:58:40,820 DEBUG [RS_FLUSH_OPERATIONS-regionserver/037c58a19ddb:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=43 2024-11-16T09:58:40,821 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37877 {}] master.HMaster(4169): Remote procedure done, pid=43 2024-11-16T09:58:40,827 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=43, resume processing ppid=42 2024-11-16T09:58:40,827 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=43, ppid=42, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.3380 sec 2024-11-16T09:58:40,832 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=42, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=42, table=TestAcidGuarantees in 2.3490 sec 2024-11-16T09:58:41,237 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] regionserver.HRegion(8855): Flush requested on 7c7f87a6124bfe2f4049ee7803e0f98c 2024-11-16T09:58:41,237 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-16T09:58:41,238 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 7c7f87a6124bfe2f4049ee7803e0f98c 3/3 column families, dataSize=167.72 KB heapSize=440.20 KB 2024-11-16T09:58:41,250 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241116e2f07f3fb6374ae8abf676c9d3395488_7c7f87a6124bfe2f4049ee7803e0f98c is 50, key is test_row_0/A:col10/1731751121236/Put/seqid=0 2024-11-16T09:58:41,254 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c7f87a6124bfe2f4049ee7803e0f98c, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T09:58:41,254 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52392 deadline: 1731751181251, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c7f87a6124bfe2f4049ee7803e0f98c, server=037c58a19ddb,37267,1731751077844 2024-11-16T09:58:41,255 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731751108989.7c7f87a6124bfe2f4049ee7803e0f98c., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 , the old value is region=TestAcidGuarantees,,1731751108989.7c7f87a6124bfe2f4049ee7803e0f98c., hostname=037c58a19ddb,37267,1731751077844, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c7f87a6124bfe2f4049ee7803e0f98c, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:58:41,255 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731751108989.7c7f87a6124bfe2f4049ee7803e0f98c., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c7f87a6124bfe2f4049ee7803e0f98c, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:58:41,255 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731751108989.7c7f87a6124bfe2f4049ee7803e0f98c., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 because the exception is null or not the one we care about 2024-11-16T09:58:41,258 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c7f87a6124bfe2f4049ee7803e0f98c, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T09:58:41,258 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52396 deadline: 1731751181254, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c7f87a6124bfe2f4049ee7803e0f98c, server=037c58a19ddb,37267,1731751077844 2024-11-16T09:58:41,259 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731751108989.7c7f87a6124bfe2f4049ee7803e0f98c., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 , the old value is region=TestAcidGuarantees,,1731751108989.7c7f87a6124bfe2f4049ee7803e0f98c., hostname=037c58a19ddb,37267,1731751077844, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c7f87a6124bfe2f4049ee7803e0f98c, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:58:41,259 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731751108989.7c7f87a6124bfe2f4049ee7803e0f98c., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c7f87a6124bfe2f4049ee7803e0f98c, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:58:41,259 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731751108989.7c7f87a6124bfe2f4049ee7803e0f98c., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 because the exception is null or not the one we care about 2024-11-16T09:58:41,259 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c7f87a6124bfe2f4049ee7803e0f98c, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T09:58:41,259 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52400 deadline: 1731751181254, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c7f87a6124bfe2f4049ee7803e0f98c, server=037c58a19ddb,37267,1731751077844 2024-11-16T09:58:41,259 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c7f87a6124bfe2f4049ee7803e0f98c, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T09:58:41,259 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731751108989.7c7f87a6124bfe2f4049ee7803e0f98c., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 , the old value is region=TestAcidGuarantees,,1731751108989.7c7f87a6124bfe2f4049ee7803e0f98c., hostname=037c58a19ddb,37267,1731751077844, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c7f87a6124bfe2f4049ee7803e0f98c, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:58:41,259 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731751108989.7c7f87a6124bfe2f4049ee7803e0f98c., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c7f87a6124bfe2f4049ee7803e0f98c, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:58:41,259 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52420 deadline: 1731751181254, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c7f87a6124bfe2f4049ee7803e0f98c, server=037c58a19ddb,37267,1731751077844 2024-11-16T09:58:41,259 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731751108989.7c7f87a6124bfe2f4049ee7803e0f98c., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 because the exception is null or not the one we care about 2024-11-16T09:58:41,260 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731751108989.7c7f87a6124bfe2f4049ee7803e0f98c., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 , the old value is region=TestAcidGuarantees,,1731751108989.7c7f87a6124bfe2f4049ee7803e0f98c., hostname=037c58a19ddb,37267,1731751077844, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c7f87a6124bfe2f4049ee7803e0f98c, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:58:41,260 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731751108989.7c7f87a6124bfe2f4049ee7803e0f98c., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c7f87a6124bfe2f4049ee7803e0f98c, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:58:41,260 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731751108989.7c7f87a6124bfe2f4049ee7803e0f98c., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 because the exception is null or not the one we care about 2024-11-16T09:58:41,262 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37591 is added to blk_1073742009_1185 (size=14794) 2024-11-16T09:58:41,264 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T09:58:41,278 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241116e2f07f3fb6374ae8abf676c9d3395488_7c7f87a6124bfe2f4049ee7803e0f98c to hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241116e2f07f3fb6374ae8abf676c9d3395488_7c7f87a6124bfe2f4049ee7803e0f98c 2024-11-16T09:58:41,279 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/7c7f87a6124bfe2f4049ee7803e0f98c/.tmp/A/2dddce9f3cd147d3b9cd2f2d801fc58a, store: [table=TestAcidGuarantees family=A region=7c7f87a6124bfe2f4049ee7803e0f98c] 2024-11-16T09:58:41,280 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/7c7f87a6124bfe2f4049ee7803e0f98c/.tmp/A/2dddce9f3cd147d3b9cd2f2d801fc58a is 175, key is test_row_0/A:col10/1731751121236/Put/seqid=0 2024-11-16T09:58:41,288 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37591 is added to blk_1073742010_1186 (size=39749) 2024-11-16T09:58:41,291 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=173, memsize=58.1 K, hasBloomFilter=true, into tmp file hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/7c7f87a6124bfe2f4049ee7803e0f98c/.tmp/A/2dddce9f3cd147d3b9cd2f2d801fc58a 2024-11-16T09:58:41,301 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/7c7f87a6124bfe2f4049ee7803e0f98c/.tmp/B/cb196474d7ac4eaf876dc872b6eed2f4 is 50, key is test_row_0/B:col10/1731751121236/Put/seqid=0 2024-11-16T09:58:41,319 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37591 is added to blk_1073742011_1187 (size=12151) 2024-11-16T09:58:41,323 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=58.14 KB at sequenceid=173 (bloomFilter=true), to=hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/7c7f87a6124bfe2f4049ee7803e0f98c/.tmp/B/cb196474d7ac4eaf876dc872b6eed2f4 2024-11-16T09:58:41,339 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/7c7f87a6124bfe2f4049ee7803e0f98c/.tmp/C/2f2c7dd9248d4963b68fdbffbc6a8106 is 50, key is test_row_0/C:col10/1731751121236/Put/seqid=0 2024-11-16T09:58:41,366 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c7f87a6124bfe2f4049ee7803e0f98c, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T09:58:41,366 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52400 deadline: 1731751181363, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c7f87a6124bfe2f4049ee7803e0f98c, server=037c58a19ddb,37267,1731751077844 2024-11-16T09:58:41,367 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731751108989.7c7f87a6124bfe2f4049ee7803e0f98c., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 , the old value is region=TestAcidGuarantees,,1731751108989.7c7f87a6124bfe2f4049ee7803e0f98c., hostname=037c58a19ddb,37267,1731751077844, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c7f87a6124bfe2f4049ee7803e0f98c, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:58:41,367 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731751108989.7c7f87a6124bfe2f4049ee7803e0f98c., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c7f87a6124bfe2f4049ee7803e0f98c, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:58:41,367 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c7f87a6124bfe2f4049ee7803e0f98c, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T09:58:41,367 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731751108989.7c7f87a6124bfe2f4049ee7803e0f98c., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 because the exception is null or not the one we care about 2024-11-16T09:58:41,367 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52420 deadline: 1731751181363, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c7f87a6124bfe2f4049ee7803e0f98c, server=037c58a19ddb,37267,1731751077844 2024-11-16T09:58:41,367 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731751108989.7c7f87a6124bfe2f4049ee7803e0f98c., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 , the old value is region=TestAcidGuarantees,,1731751108989.7c7f87a6124bfe2f4049ee7803e0f98c., hostname=037c58a19ddb,37267,1731751077844, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c7f87a6124bfe2f4049ee7803e0f98c, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:58:41,367 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731751108989.7c7f87a6124bfe2f4049ee7803e0f98c., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c7f87a6124bfe2f4049ee7803e0f98c, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:58:41,367 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731751108989.7c7f87a6124bfe2f4049ee7803e0f98c., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 because the exception is null or not the one we care about 2024-11-16T09:58:41,367 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c7f87a6124bfe2f4049ee7803e0f98c, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T09:58:41,367 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52396 deadline: 1731751181363, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c7f87a6124bfe2f4049ee7803e0f98c, server=037c58a19ddb,37267,1731751077844 2024-11-16T09:58:41,368 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731751108989.7c7f87a6124bfe2f4049ee7803e0f98c., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 , the old value is region=TestAcidGuarantees,,1731751108989.7c7f87a6124bfe2f4049ee7803e0f98c., hostname=037c58a19ddb,37267,1731751077844, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c7f87a6124bfe2f4049ee7803e0f98c, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:58:41,368 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731751108989.7c7f87a6124bfe2f4049ee7803e0f98c., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c7f87a6124bfe2f4049ee7803e0f98c, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:58:41,368 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731751108989.7c7f87a6124bfe2f4049ee7803e0f98c., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 because the exception is null or not the one we care about 2024-11-16T09:58:41,368 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c7f87a6124bfe2f4049ee7803e0f98c, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T09:58:41,368 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52392 deadline: 1731751181363, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c7f87a6124bfe2f4049ee7803e0f98c, server=037c58a19ddb,37267,1731751077844 2024-11-16T09:58:41,368 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731751108989.7c7f87a6124bfe2f4049ee7803e0f98c., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 , the old value is region=TestAcidGuarantees,,1731751108989.7c7f87a6124bfe2f4049ee7803e0f98c., hostname=037c58a19ddb,37267,1731751077844, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c7f87a6124bfe2f4049ee7803e0f98c, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:58:41,368 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731751108989.7c7f87a6124bfe2f4049ee7803e0f98c., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c7f87a6124bfe2f4049ee7803e0f98c, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:58:41,368 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731751108989.7c7f87a6124bfe2f4049ee7803e0f98c., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 because the exception is null or not the one we care about 2024-11-16T09:58:41,378 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37591 is added to blk_1073742012_1188 (size=12151) 2024-11-16T09:58:41,378 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=58.14 KB at sequenceid=173 (bloomFilter=true), to=hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/7c7f87a6124bfe2f4049ee7803e0f98c/.tmp/C/2f2c7dd9248d4963b68fdbffbc6a8106 2024-11-16T09:58:41,391 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/7c7f87a6124bfe2f4049ee7803e0f98c/.tmp/A/2dddce9f3cd147d3b9cd2f2d801fc58a as hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/7c7f87a6124bfe2f4049ee7803e0f98c/A/2dddce9f3cd147d3b9cd2f2d801fc58a 2024-11-16T09:58:41,396 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/7c7f87a6124bfe2f4049ee7803e0f98c/A/2dddce9f3cd147d3b9cd2f2d801fc58a, entries=200, sequenceid=173, filesize=38.8 K 2024-11-16T09:58:41,397 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/7c7f87a6124bfe2f4049ee7803e0f98c/.tmp/B/cb196474d7ac4eaf876dc872b6eed2f4 as hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/7c7f87a6124bfe2f4049ee7803e0f98c/B/cb196474d7ac4eaf876dc872b6eed2f4 2024-11-16T09:58:41,403 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/7c7f87a6124bfe2f4049ee7803e0f98c/B/cb196474d7ac4eaf876dc872b6eed2f4, entries=150, sequenceid=173, filesize=11.9 K 2024-11-16T09:58:41,404 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/7c7f87a6124bfe2f4049ee7803e0f98c/.tmp/C/2f2c7dd9248d4963b68fdbffbc6a8106 as hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/7c7f87a6124bfe2f4049ee7803e0f98c/C/2f2c7dd9248d4963b68fdbffbc6a8106 2024-11-16T09:58:41,410 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/7c7f87a6124bfe2f4049ee7803e0f98c/C/2f2c7dd9248d4963b68fdbffbc6a8106, entries=150, sequenceid=173, filesize=11.9 K 2024-11-16T09:58:41,412 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~174.43 KB/178620, heapSize ~457.73 KB/468720, currentSize=40.25 KB/41220 for 7c7f87a6124bfe2f4049ee7803e0f98c in 174ms, sequenceid=173, compaction requested=true 2024-11-16T09:58:41,412 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 7c7f87a6124bfe2f4049ee7803e0f98c: 2024-11-16T09:58:41,412 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 7c7f87a6124bfe2f4049ee7803e0f98c:A, priority=-2147483648, current under compaction store size is 1 2024-11-16T09:58:41,412 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-16T09:58:41,412 DEBUG [RS:0;037c58a19ddb:37267-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-16T09:58:41,412 DEBUG [RS:0;037c58a19ddb:37267-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-16T09:58:41,413 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 7c7f87a6124bfe2f4049ee7803e0f98c:B, priority=-2147483648, current under compaction store size is 2 2024-11-16T09:58:41,413 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-16T09:58:41,413 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 7c7f87a6124bfe2f4049ee7803e0f98c:C, priority=-2147483648, current under compaction store size is 3 2024-11-16T09:58:41,413 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-16T09:58:41,414 DEBUG [RS:0;037c58a19ddb:37267-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48594 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-16T09:58:41,414 DEBUG [RS:0;037c58a19ddb:37267-longCompactions-0 {}] regionserver.HStore(1541): 7c7f87a6124bfe2f4049ee7803e0f98c/B is initiating minor compaction (all files) 2024-11-16T09:58:41,414 INFO [RS:0;037c58a19ddb:37267-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 7c7f87a6124bfe2f4049ee7803e0f98c/B in TestAcidGuarantees,,1731751108989.7c7f87a6124bfe2f4049ee7803e0f98c. 2024-11-16T09:58:41,414 INFO [RS:0;037c58a19ddb:37267-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/7c7f87a6124bfe2f4049ee7803e0f98c/B/61c3f1d80f724fefaa1bfbedddebbb01, hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/7c7f87a6124bfe2f4049ee7803e0f98c/B/ed2ff962b6c54b2e95ec44bbec83832f, hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/7c7f87a6124bfe2f4049ee7803e0f98c/B/df3050c6104a4f5a843ce0d720c0bd8c, hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/7c7f87a6124bfe2f4049ee7803e0f98c/B/cb196474d7ac4eaf876dc872b6eed2f4] into tmpdir=hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/7c7f87a6124bfe2f4049ee7803e0f98c/.tmp, totalSize=47.5 K 2024-11-16T09:58:41,415 DEBUG [RS:0;037c58a19ddb:37267-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 133054 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-16T09:58:41,415 DEBUG [RS:0;037c58a19ddb:37267-longCompactions-0 {}] compactions.Compactor(225): Compacting 61c3f1d80f724fefaa1bfbedddebbb01, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=103, earliestPutTs=1731751117995 2024-11-16T09:58:41,415 DEBUG [RS:0;037c58a19ddb:37267-shortCompactions-0 {}] regionserver.HStore(1541): 7c7f87a6124bfe2f4049ee7803e0f98c/A is initiating minor compaction (all files) 2024-11-16T09:58:41,415 INFO [RS:0;037c58a19ddb:37267-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 7c7f87a6124bfe2f4049ee7803e0f98c/A in TestAcidGuarantees,,1731751108989.7c7f87a6124bfe2f4049ee7803e0f98c. 2024-11-16T09:58:41,415 INFO [RS:0;037c58a19ddb:37267-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/7c7f87a6124bfe2f4049ee7803e0f98c/A/4ca189b9a842483db6e475afcc8899aa, hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/7c7f87a6124bfe2f4049ee7803e0f98c/A/64fcc2a7d41e439a952ea7e0969d6e41, hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/7c7f87a6124bfe2f4049ee7803e0f98c/A/8c843a2fb4074619b6de587720d98901, hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/7c7f87a6124bfe2f4049ee7803e0f98c/A/2dddce9f3cd147d3b9cd2f2d801fc58a] into tmpdir=hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/7c7f87a6124bfe2f4049ee7803e0f98c/.tmp, totalSize=129.9 K 2024-11-16T09:58:41,415 INFO [RS:0;037c58a19ddb:37267-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=12 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731751108989.7c7f87a6124bfe2f4049ee7803e0f98c. 2024-11-16T09:58:41,415 DEBUG [RS:0;037c58a19ddb:37267-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731751108989.7c7f87a6124bfe2f4049ee7803e0f98c. files: [hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/7c7f87a6124bfe2f4049ee7803e0f98c/A/4ca189b9a842483db6e475afcc8899aa, hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/7c7f87a6124bfe2f4049ee7803e0f98c/A/64fcc2a7d41e439a952ea7e0969d6e41, hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/7c7f87a6124bfe2f4049ee7803e0f98c/A/8c843a2fb4074619b6de587720d98901, hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/7c7f87a6124bfe2f4049ee7803e0f98c/A/2dddce9f3cd147d3b9cd2f2d801fc58a] 2024-11-16T09:58:41,415 DEBUG [RS:0;037c58a19ddb:37267-longCompactions-0 {}] compactions.Compactor(225): Compacting ed2ff962b6c54b2e95ec44bbec83832f, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=131, earliestPutTs=1731751118146 2024-11-16T09:58:41,415 DEBUG [RS:0;037c58a19ddb:37267-shortCompactions-0 {}] compactions.Compactor(225): Compacting 4ca189b9a842483db6e475afcc8899aa, keycount=150, bloomtype=ROW, size=30.5 K, encoding=NONE, compression=NONE, seqNum=103, earliestPutTs=1731751117995 2024-11-16T09:58:41,416 DEBUG [RS:0;037c58a19ddb:37267-longCompactions-0 {}] compactions.Compactor(225): Compacting df3050c6104a4f5a843ce0d720c0bd8c, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=143, earliestPutTs=1731751118795 2024-11-16T09:58:41,416 DEBUG [RS:0;037c58a19ddb:37267-shortCompactions-0 {}] compactions.Compactor(225): Compacting 64fcc2a7d41e439a952ea7e0969d6e41, keycount=150, bloomtype=ROW, size=30.3 K, encoding=NONE, compression=NONE, seqNum=131, earliestPutTs=1731751118146 2024-11-16T09:58:41,416 DEBUG [RS:0;037c58a19ddb:37267-shortCompactions-0 {}] compactions.Compactor(225): Compacting 8c843a2fb4074619b6de587720d98901, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=143, earliestPutTs=1731751118795 2024-11-16T09:58:41,417 DEBUG [RS:0;037c58a19ddb:37267-longCompactions-0 {}] compactions.Compactor(225): Compacting cb196474d7ac4eaf876dc872b6eed2f4, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=173, earliestPutTs=1731751119982 2024-11-16T09:58:41,417 DEBUG [RS:0;037c58a19ddb:37267-shortCompactions-0 {}] compactions.Compactor(225): Compacting 2dddce9f3cd147d3b9cd2f2d801fc58a, keycount=200, bloomtype=ROW, size=38.8 K, encoding=NONE, compression=NONE, seqNum=173, earliestPutTs=1731751119982 2024-11-16T09:58:41,431 INFO [RS:0;037c58a19ddb:37267-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=7c7f87a6124bfe2f4049ee7803e0f98c] 2024-11-16T09:58:41,435 DEBUG [RS:0;037c58a19ddb:37267-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241116517d5bf87cd941cf81a4fb11510a1675_7c7f87a6124bfe2f4049ee7803e0f98c store=[table=TestAcidGuarantees family=A region=7c7f87a6124bfe2f4049ee7803e0f98c] 2024-11-16T09:58:41,436 INFO [RS:0;037c58a19ddb:37267-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 7c7f87a6124bfe2f4049ee7803e0f98c#B#compaction#163 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-16T09:58:41,437 DEBUG [RS:0;037c58a19ddb:37267-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/7c7f87a6124bfe2f4049ee7803e0f98c/.tmp/B/2bf5259150774f07a21de5b3fb39be24 is 50, key is test_row_0/B:col10/1731751121236/Put/seqid=0 2024-11-16T09:58:41,438 DEBUG [RS:0;037c58a19ddb:37267-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241116517d5bf87cd941cf81a4fb11510a1675_7c7f87a6124bfe2f4049ee7803e0f98c, store=[table=TestAcidGuarantees family=A region=7c7f87a6124bfe2f4049ee7803e0f98c] 2024-11-16T09:58:41,438 DEBUG [RS:0;037c58a19ddb:37267-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241116517d5bf87cd941cf81a4fb11510a1675_7c7f87a6124bfe2f4049ee7803e0f98c because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=7c7f87a6124bfe2f4049ee7803e0f98c] 2024-11-16T09:58:41,456 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37591 is added to blk_1073742014_1190 (size=4469) 2024-11-16T09:58:41,457 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37591 is added to blk_1073742013_1189 (size=12527) 2024-11-16T09:58:41,457 INFO [RS:0;037c58a19ddb:37267-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 7c7f87a6124bfe2f4049ee7803e0f98c#A#compaction#162 average throughput is 0.94 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-16T09:58:41,458 DEBUG [RS:0;037c58a19ddb:37267-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/7c7f87a6124bfe2f4049ee7803e0f98c/.tmp/A/3bff852129eb44848826f920130a1069 is 175, key is test_row_0/A:col10/1731751121236/Put/seqid=0 2024-11-16T09:58:41,479 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37591 is added to blk_1073742015_1191 (size=31481) 2024-11-16T09:58:41,487 DEBUG [RS:0;037c58a19ddb:37267-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/7c7f87a6124bfe2f4049ee7803e0f98c/.tmp/A/3bff852129eb44848826f920130a1069 as hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/7c7f87a6124bfe2f4049ee7803e0f98c/A/3bff852129eb44848826f920130a1069 2024-11-16T09:58:41,501 INFO [RS:0;037c58a19ddb:37267-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 7c7f87a6124bfe2f4049ee7803e0f98c/A of 7c7f87a6124bfe2f4049ee7803e0f98c into 3bff852129eb44848826f920130a1069(size=30.7 K), total size for store is 30.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-16T09:58:41,501 DEBUG [RS:0;037c58a19ddb:37267-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 7c7f87a6124bfe2f4049ee7803e0f98c: 2024-11-16T09:58:41,502 INFO [RS:0;037c58a19ddb:37267-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731751108989.7c7f87a6124bfe2f4049ee7803e0f98c., storeName=7c7f87a6124bfe2f4049ee7803e0f98c/A, priority=12, startTime=1731751121412; duration=0sec 2024-11-16T09:58:41,502 DEBUG [RS:0;037c58a19ddb:37267-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-16T09:58:41,502 DEBUG [RS:0;037c58a19ddb:37267-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 7c7f87a6124bfe2f4049ee7803e0f98c:A 2024-11-16T09:58:41,502 DEBUG [RS:0;037c58a19ddb:37267-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-16T09:58:41,503 DEBUG [RS:0;037c58a19ddb:37267-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36627 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-16T09:58:41,503 DEBUG [RS:0;037c58a19ddb:37267-shortCompactions-0 {}] regionserver.HStore(1541): 7c7f87a6124bfe2f4049ee7803e0f98c/C is initiating minor compaction (all files) 2024-11-16T09:58:41,503 INFO [RS:0;037c58a19ddb:37267-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 7c7f87a6124bfe2f4049ee7803e0f98c/C in TestAcidGuarantees,,1731751108989.7c7f87a6124bfe2f4049ee7803e0f98c. 2024-11-16T09:58:41,503 INFO [RS:0;037c58a19ddb:37267-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/7c7f87a6124bfe2f4049ee7803e0f98c/C/2e3828d7d79b488a9c4d75f3a4e6af2d, hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/7c7f87a6124bfe2f4049ee7803e0f98c/C/cd3257160203416e884bed4bb651ac18, hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/7c7f87a6124bfe2f4049ee7803e0f98c/C/2f2c7dd9248d4963b68fdbffbc6a8106] into tmpdir=hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/7c7f87a6124bfe2f4049ee7803e0f98c/.tmp, totalSize=35.8 K 2024-11-16T09:58:41,504 DEBUG [RS:0;037c58a19ddb:37267-shortCompactions-0 {}] compactions.Compactor(225): Compacting 2e3828d7d79b488a9c4d75f3a4e6af2d, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=131, earliestPutTs=1731751118146 2024-11-16T09:58:41,504 DEBUG [RS:0;037c58a19ddb:37267-shortCompactions-0 {}] compactions.Compactor(225): Compacting cd3257160203416e884bed4bb651ac18, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=143, earliestPutTs=1731751118795 2024-11-16T09:58:41,505 DEBUG [RS:0;037c58a19ddb:37267-shortCompactions-0 {}] compactions.Compactor(225): Compacting 2f2c7dd9248d4963b68fdbffbc6a8106, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=173, earliestPutTs=1731751119982 2024-11-16T09:58:41,530 INFO [RS:0;037c58a19ddb:37267-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 7c7f87a6124bfe2f4049ee7803e0f98c#C#compaction#164 average throughput is 1.31 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-16T09:58:41,530 DEBUG [RS:0;037c58a19ddb:37267-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/7c7f87a6124bfe2f4049ee7803e0f98c/.tmp/C/20dd8e2f925044debf6455fa91c2494b is 50, key is test_row_0/C:col10/1731751121236/Put/seqid=0 2024-11-16T09:58:41,553 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37591 is added to blk_1073742016_1192 (size=12527) 2024-11-16T09:58:41,564 DEBUG [RS:0;037c58a19ddb:37267-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/7c7f87a6124bfe2f4049ee7803e0f98c/.tmp/C/20dd8e2f925044debf6455fa91c2494b as hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/7c7f87a6124bfe2f4049ee7803e0f98c/C/20dd8e2f925044debf6455fa91c2494b 2024-11-16T09:58:41,571 INFO [RS:0;037c58a19ddb:37267-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 7c7f87a6124bfe2f4049ee7803e0f98c/C of 7c7f87a6124bfe2f4049ee7803e0f98c into 20dd8e2f925044debf6455fa91c2494b(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-16T09:58:41,571 DEBUG [RS:0;037c58a19ddb:37267-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 7c7f87a6124bfe2f4049ee7803e0f98c: 2024-11-16T09:58:41,571 INFO [RS:0;037c58a19ddb:37267-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731751108989.7c7f87a6124bfe2f4049ee7803e0f98c., storeName=7c7f87a6124bfe2f4049ee7803e0f98c/C, priority=13, startTime=1731751121413; duration=0sec 2024-11-16T09:58:41,571 DEBUG [RS:0;037c58a19ddb:37267-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-16T09:58:41,571 DEBUG [RS:0;037c58a19ddb:37267-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 7c7f87a6124bfe2f4049ee7803e0f98c:C 2024-11-16T09:58:41,577 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-16T09:58:41,578 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 7c7f87a6124bfe2f4049ee7803e0f98c 3/3 column families, dataSize=53.67 KB heapSize=141.38 KB 2024-11-16T09:58:41,578 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] regionserver.HRegion(8855): Flush requested on 7c7f87a6124bfe2f4049ee7803e0f98c 2024-11-16T09:58:41,616 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241116da89ba3df1fe4b6eb99a016faaa5721d_7c7f87a6124bfe2f4049ee7803e0f98c is 50, key is test_row_0/A:col10/1731751121576/Put/seqid=0 2024-11-16T09:58:41,639 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c7f87a6124bfe2f4049ee7803e0f98c, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T09:58:41,639 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52392 deadline: 1731751181633, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c7f87a6124bfe2f4049ee7803e0f98c, server=037c58a19ddb,37267,1731751077844 2024-11-16T09:58:41,639 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731751108989.7c7f87a6124bfe2f4049ee7803e0f98c., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 , the old value is region=TestAcidGuarantees,,1731751108989.7c7f87a6124bfe2f4049ee7803e0f98c., hostname=037c58a19ddb,37267,1731751077844, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c7f87a6124bfe2f4049ee7803e0f98c, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:58:41,639 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731751108989.7c7f87a6124bfe2f4049ee7803e0f98c., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c7f87a6124bfe2f4049ee7803e0f98c, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:58:41,639 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c7f87a6124bfe2f4049ee7803e0f98c, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T09:58:41,639 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731751108989.7c7f87a6124bfe2f4049ee7803e0f98c., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 because the exception is null or not the one we care about 2024-11-16T09:58:41,640 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52420 deadline: 1731751181633, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c7f87a6124bfe2f4049ee7803e0f98c, server=037c58a19ddb,37267,1731751077844 2024-11-16T09:58:41,640 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731751108989.7c7f87a6124bfe2f4049ee7803e0f98c., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 , the old value is region=TestAcidGuarantees,,1731751108989.7c7f87a6124bfe2f4049ee7803e0f98c., hostname=037c58a19ddb,37267,1731751077844, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c7f87a6124bfe2f4049ee7803e0f98c, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:58:41,640 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731751108989.7c7f87a6124bfe2f4049ee7803e0f98c., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c7f87a6124bfe2f4049ee7803e0f98c, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:58:41,640 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731751108989.7c7f87a6124bfe2f4049ee7803e0f98c., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 because the exception is null or not the one we care about 2024-11-16T09:58:41,642 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c7f87a6124bfe2f4049ee7803e0f98c, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T09:58:41,642 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] ipc.CallRunner(138): callId: 71 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52396 deadline: 1731751181635, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c7f87a6124bfe2f4049ee7803e0f98c, server=037c58a19ddb,37267,1731751077844 2024-11-16T09:58:41,643 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731751108989.7c7f87a6124bfe2f4049ee7803e0f98c., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 , the old value is region=TestAcidGuarantees,,1731751108989.7c7f87a6124bfe2f4049ee7803e0f98c., hostname=037c58a19ddb,37267,1731751077844, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c7f87a6124bfe2f4049ee7803e0f98c, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:58:41,643 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731751108989.7c7f87a6124bfe2f4049ee7803e0f98c., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c7f87a6124bfe2f4049ee7803e0f98c, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:58:41,643 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731751108989.7c7f87a6124bfe2f4049ee7803e0f98c., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 because the exception is null or not the one we care about 2024-11-16T09:58:41,643 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c7f87a6124bfe2f4049ee7803e0f98c, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T09:58:41,643 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] ipc.CallRunner(138): callId: 72 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52400 deadline: 1731751181637, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c7f87a6124bfe2f4049ee7803e0f98c, server=037c58a19ddb,37267,1731751077844 2024-11-16T09:58:41,644 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731751108989.7c7f87a6124bfe2f4049ee7803e0f98c., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 , the old value is region=TestAcidGuarantees,,1731751108989.7c7f87a6124bfe2f4049ee7803e0f98c., hostname=037c58a19ddb,37267,1731751077844, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c7f87a6124bfe2f4049ee7803e0f98c, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:58:41,644 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731751108989.7c7f87a6124bfe2f4049ee7803e0f98c., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c7f87a6124bfe2f4049ee7803e0f98c, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:58:41,644 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731751108989.7c7f87a6124bfe2f4049ee7803e0f98c., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 because the exception is null or not the one we care about 2024-11-16T09:58:41,665 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37591 is added to blk_1073742017_1193 (size=19774) 2024-11-16T09:58:41,674 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T09:58:41,678 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241116da89ba3df1fe4b6eb99a016faaa5721d_7c7f87a6124bfe2f4049ee7803e0f98c to hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241116da89ba3df1fe4b6eb99a016faaa5721d_7c7f87a6124bfe2f4049ee7803e0f98c 2024-11-16T09:58:41,679 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/7c7f87a6124bfe2f4049ee7803e0f98c/.tmp/A/42e604f249434caf82bcc0dc3532e984, store: [table=TestAcidGuarantees family=A region=7c7f87a6124bfe2f4049ee7803e0f98c] 2024-11-16T09:58:41,680 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/7c7f87a6124bfe2f4049ee7803e0f98c/.tmp/A/42e604f249434caf82bcc0dc3532e984 is 175, key is test_row_0/A:col10/1731751121576/Put/seqid=0 2024-11-16T09:58:41,689 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37591 is added to blk_1073742018_1194 (size=57033) 2024-11-16T09:58:41,749 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c7f87a6124bfe2f4049ee7803e0f98c, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T09:58:41,750 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52420 deadline: 1731751181743, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c7f87a6124bfe2f4049ee7803e0f98c, server=037c58a19ddb,37267,1731751077844 2024-11-16T09:58:41,750 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731751108989.7c7f87a6124bfe2f4049ee7803e0f98c., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 , the old value is region=TestAcidGuarantees,,1731751108989.7c7f87a6124bfe2f4049ee7803e0f98c., hostname=037c58a19ddb,37267,1731751077844, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c7f87a6124bfe2f4049ee7803e0f98c, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:58:41,750 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731751108989.7c7f87a6124bfe2f4049ee7803e0f98c., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c7f87a6124bfe2f4049ee7803e0f98c, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:58:41,750 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731751108989.7c7f87a6124bfe2f4049ee7803e0f98c., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 because the exception is null or not the one we care about 2024-11-16T09:58:41,750 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c7f87a6124bfe2f4049ee7803e0f98c, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T09:58:41,750 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52392 deadline: 1731751181743, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c7f87a6124bfe2f4049ee7803e0f98c, server=037c58a19ddb,37267,1731751077844 2024-11-16T09:58:41,751 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731751108989.7c7f87a6124bfe2f4049ee7803e0f98c., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 , the old value is region=TestAcidGuarantees,,1731751108989.7c7f87a6124bfe2f4049ee7803e0f98c., hostname=037c58a19ddb,37267,1731751077844, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c7f87a6124bfe2f4049ee7803e0f98c, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:58:41,751 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731751108989.7c7f87a6124bfe2f4049ee7803e0f98c., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c7f87a6124bfe2f4049ee7803e0f98c, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:58:41,751 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731751108989.7c7f87a6124bfe2f4049ee7803e0f98c., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 because the exception is null or not the one we care about 2024-11-16T09:58:41,754 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c7f87a6124bfe2f4049ee7803e0f98c, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T09:58:41,754 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] ipc.CallRunner(138): callId: 72 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52396 deadline: 1731751181752, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c7f87a6124bfe2f4049ee7803e0f98c, server=037c58a19ddb,37267,1731751077844 2024-11-16T09:58:41,754 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731751108989.7c7f87a6124bfe2f4049ee7803e0f98c., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 , the old value is region=TestAcidGuarantees,,1731751108989.7c7f87a6124bfe2f4049ee7803e0f98c., hostname=037c58a19ddb,37267,1731751077844, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c7f87a6124bfe2f4049ee7803e0f98c, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:58:41,754 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731751108989.7c7f87a6124bfe2f4049ee7803e0f98c., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c7f87a6124bfe2f4049ee7803e0f98c, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:58:41,754 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731751108989.7c7f87a6124bfe2f4049ee7803e0f98c., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 because the exception is null or not the one we care about 2024-11-16T09:58:41,755 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c7f87a6124bfe2f4049ee7803e0f98c, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T09:58:41,756 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] ipc.CallRunner(138): callId: 73 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52400 deadline: 1731751181753, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c7f87a6124bfe2f4049ee7803e0f98c, server=037c58a19ddb,37267,1731751077844 2024-11-16T09:58:41,756 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731751108989.7c7f87a6124bfe2f4049ee7803e0f98c., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 , the old value is region=TestAcidGuarantees,,1731751108989.7c7f87a6124bfe2f4049ee7803e0f98c., hostname=037c58a19ddb,37267,1731751077844, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c7f87a6124bfe2f4049ee7803e0f98c, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:58:41,756 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731751108989.7c7f87a6124bfe2f4049ee7803e0f98c., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c7f87a6124bfe2f4049ee7803e0f98c, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:58:41,756 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731751108989.7c7f87a6124bfe2f4049ee7803e0f98c., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 because the exception is null or not the one we care about 2024-11-16T09:58:41,873 DEBUG [RS:0;037c58a19ddb:37267-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/7c7f87a6124bfe2f4049ee7803e0f98c/.tmp/B/2bf5259150774f07a21de5b3fb39be24 as hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/7c7f87a6124bfe2f4049ee7803e0f98c/B/2bf5259150774f07a21de5b3fb39be24 2024-11-16T09:58:41,880 INFO [RS:0;037c58a19ddb:37267-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 7c7f87a6124bfe2f4049ee7803e0f98c/B of 7c7f87a6124bfe2f4049ee7803e0f98c into 2bf5259150774f07a21de5b3fb39be24(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-16T09:58:41,880 DEBUG [RS:0;037c58a19ddb:37267-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 7c7f87a6124bfe2f4049ee7803e0f98c: 2024-11-16T09:58:41,880 INFO [RS:0;037c58a19ddb:37267-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731751108989.7c7f87a6124bfe2f4049ee7803e0f98c., storeName=7c7f87a6124bfe2f4049ee7803e0f98c/B, priority=12, startTime=1731751121412; duration=0sec 2024-11-16T09:58:41,880 DEBUG [RS:0;037c58a19ddb:37267-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-16T09:58:41,881 DEBUG [RS:0;037c58a19ddb:37267-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 7c7f87a6124bfe2f4049ee7803e0f98c:B 2024-11-16T09:58:41,979 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c7f87a6124bfe2f4049ee7803e0f98c, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T09:58:41,980 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52392 deadline: 1731751181978, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c7f87a6124bfe2f4049ee7803e0f98c, server=037c58a19ddb,37267,1731751077844 2024-11-16T09:58:41,980 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731751108989.7c7f87a6124bfe2f4049ee7803e0f98c., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 , the old value is region=TestAcidGuarantees,,1731751108989.7c7f87a6124bfe2f4049ee7803e0f98c., hostname=037c58a19ddb,37267,1731751077844, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c7f87a6124bfe2f4049ee7803e0f98c, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:58:41,980 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731751108989.7c7f87a6124bfe2f4049ee7803e0f98c., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c7f87a6124bfe2f4049ee7803e0f98c, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:58:41,980 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c7f87a6124bfe2f4049ee7803e0f98c, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T09:58:41,980 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731751108989.7c7f87a6124bfe2f4049ee7803e0f98c., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 because the exception is null or not the one we care about 2024-11-16T09:58:41,981 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52420 deadline: 1731751181978, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c7f87a6124bfe2f4049ee7803e0f98c, server=037c58a19ddb,37267,1731751077844 2024-11-16T09:58:41,981 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c7f87a6124bfe2f4049ee7803e0f98c, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T09:58:41,981 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731751108989.7c7f87a6124bfe2f4049ee7803e0f98c., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 , the old value is region=TestAcidGuarantees,,1731751108989.7c7f87a6124bfe2f4049ee7803e0f98c., hostname=037c58a19ddb,37267,1731751077844, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c7f87a6124bfe2f4049ee7803e0f98c, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:58:41,981 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731751108989.7c7f87a6124bfe2f4049ee7803e0f98c., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c7f87a6124bfe2f4049ee7803e0f98c, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:58:41,981 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] ipc.CallRunner(138): callId: 74 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52400 deadline: 1731751181978, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c7f87a6124bfe2f4049ee7803e0f98c, server=037c58a19ddb,37267,1731751077844 2024-11-16T09:58:41,981 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731751108989.7c7f87a6124bfe2f4049ee7803e0f98c., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 because the exception is null or not the one we care about 2024-11-16T09:58:41,981 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731751108989.7c7f87a6124bfe2f4049ee7803e0f98c., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 , the old value is region=TestAcidGuarantees,,1731751108989.7c7f87a6124bfe2f4049ee7803e0f98c., hostname=037c58a19ddb,37267,1731751077844, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c7f87a6124bfe2f4049ee7803e0f98c, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:58:41,981 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c7f87a6124bfe2f4049ee7803e0f98c, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T09:58:41,981 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731751108989.7c7f87a6124bfe2f4049ee7803e0f98c., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c7f87a6124bfe2f4049ee7803e0f98c, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:58:41,981 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] ipc.CallRunner(138): callId: 73 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52396 deadline: 1731751181978, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c7f87a6124bfe2f4049ee7803e0f98c, server=037c58a19ddb,37267,1731751077844 2024-11-16T09:58:41,982 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731751108989.7c7f87a6124bfe2f4049ee7803e0f98c., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 , the old value is region=TestAcidGuarantees,,1731751108989.7c7f87a6124bfe2f4049ee7803e0f98c., hostname=037c58a19ddb,37267,1731751077844, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c7f87a6124bfe2f4049ee7803e0f98c, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:58:41,982 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731751108989.7c7f87a6124bfe2f4049ee7803e0f98c., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c7f87a6124bfe2f4049ee7803e0f98c, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:58:41,982 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731751108989.7c7f87a6124bfe2f4049ee7803e0f98c., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 because the exception is null or not the one we care about 2024-11-16T09:58:41,985 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731751108989.7c7f87a6124bfe2f4049ee7803e0f98c., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 because the exception is null or not the one we care about 2024-11-16T09:58:42,090 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=187, memsize=20.1 K, hasBloomFilter=true, into tmp file hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/7c7f87a6124bfe2f4049ee7803e0f98c/.tmp/A/42e604f249434caf82bcc0dc3532e984 2024-11-16T09:58:42,117 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/7c7f87a6124bfe2f4049ee7803e0f98c/.tmp/B/bc151e55d37549828dba2a7feccb86eb is 50, key is test_row_0/B:col10/1731751121576/Put/seqid=0 2024-11-16T09:58:42,149 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37591 is added to blk_1073742019_1195 (size=12151) 2024-11-16T09:58:42,153 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=187 (bloomFilter=true), to=hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/7c7f87a6124bfe2f4049ee7803e0f98c/.tmp/B/bc151e55d37549828dba2a7feccb86eb 2024-11-16T09:58:42,173 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/7c7f87a6124bfe2f4049ee7803e0f98c/.tmp/C/43bc34e4ddad4cbc85fe7916dd7fde9a is 50, key is test_row_0/C:col10/1731751121576/Put/seqid=0 2024-11-16T09:58:42,214 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37591 is added to blk_1073742020_1196 (size=12151) 2024-11-16T09:58:42,215 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=187 (bloomFilter=true), to=hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/7c7f87a6124bfe2f4049ee7803e0f98c/.tmp/C/43bc34e4ddad4cbc85fe7916dd7fde9a 2024-11-16T09:58:42,221 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/7c7f87a6124bfe2f4049ee7803e0f98c/.tmp/A/42e604f249434caf82bcc0dc3532e984 as hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/7c7f87a6124bfe2f4049ee7803e0f98c/A/42e604f249434caf82bcc0dc3532e984 2024-11-16T09:58:42,227 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/7c7f87a6124bfe2f4049ee7803e0f98c/A/42e604f249434caf82bcc0dc3532e984, entries=300, sequenceid=187, filesize=55.7 K 2024-11-16T09:58:42,228 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/7c7f87a6124bfe2f4049ee7803e0f98c/.tmp/B/bc151e55d37549828dba2a7feccb86eb as hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/7c7f87a6124bfe2f4049ee7803e0f98c/B/bc151e55d37549828dba2a7feccb86eb 2024-11-16T09:58:42,239 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/7c7f87a6124bfe2f4049ee7803e0f98c/B/bc151e55d37549828dba2a7feccb86eb, entries=150, sequenceid=187, filesize=11.9 K 2024-11-16T09:58:42,240 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/7c7f87a6124bfe2f4049ee7803e0f98c/.tmp/C/43bc34e4ddad4cbc85fe7916dd7fde9a as hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/7c7f87a6124bfe2f4049ee7803e0f98c/C/43bc34e4ddad4cbc85fe7916dd7fde9a 2024-11-16T09:58:42,245 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/7c7f87a6124bfe2f4049ee7803e0f98c/C/43bc34e4ddad4cbc85fe7916dd7fde9a, entries=150, sequenceid=187, filesize=11.9 K 2024-11-16T09:58:42,247 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~158.91 KB/162720, currentSize=140.89 KB/144270 for 7c7f87a6124bfe2f4049ee7803e0f98c in 670ms, sequenceid=187, compaction requested=false 2024-11-16T09:58:42,248 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 7c7f87a6124bfe2f4049ee7803e0f98c: 2024-11-16T09:58:42,286 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] regionserver.HRegion(8855): Flush requested on 7c7f87a6124bfe2f4049ee7803e0f98c 2024-11-16T09:58:42,286 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-16T09:58:42,286 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 7c7f87a6124bfe2f4049ee7803e0f98c 3/3 column families, dataSize=147.60 KB heapSize=387.47 KB 2024-11-16T09:58:42,306 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111650618aa9f5c94860bb796babe0a66499_7c7f87a6124bfe2f4049ee7803e0f98c is 50, key is test_row_0/A:col10/1731751121633/Put/seqid=0 2024-11-16T09:58:42,313 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c7f87a6124bfe2f4049ee7803e0f98c, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T09:58:42,313 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] ipc.CallRunner(138): callId: 76 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52396 deadline: 1731751182308, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c7f87a6124bfe2f4049ee7803e0f98c, server=037c58a19ddb,37267,1731751077844 2024-11-16T09:58:42,313 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c7f87a6124bfe2f4049ee7803e0f98c, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T09:58:42,313 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731751108989.7c7f87a6124bfe2f4049ee7803e0f98c., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 , the old value is region=TestAcidGuarantees,,1731751108989.7c7f87a6124bfe2f4049ee7803e0f98c., hostname=037c58a19ddb,37267,1731751077844, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c7f87a6124bfe2f4049ee7803e0f98c, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:58:42,313 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731751108989.7c7f87a6124bfe2f4049ee7803e0f98c., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c7f87a6124bfe2f4049ee7803e0f98c, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:58:42,313 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731751108989.7c7f87a6124bfe2f4049ee7803e0f98c., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 because the exception is null or not the one we care about 2024-11-16T09:58:42,314 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] ipc.CallRunner(138): callId: 77 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52400 deadline: 1731751182310, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c7f87a6124bfe2f4049ee7803e0f98c, server=037c58a19ddb,37267,1731751077844 2024-11-16T09:58:42,314 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c7f87a6124bfe2f4049ee7803e0f98c, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T09:58:42,314 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] ipc.CallRunner(138): callId: 73 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52420 deadline: 1731751182312, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c7f87a6124bfe2f4049ee7803e0f98c, server=037c58a19ddb,37267,1731751077844 2024-11-16T09:58:42,314 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731751108989.7c7f87a6124bfe2f4049ee7803e0f98c., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 , the old value is region=TestAcidGuarantees,,1731751108989.7c7f87a6124bfe2f4049ee7803e0f98c., hostname=037c58a19ddb,37267,1731751077844, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c7f87a6124bfe2f4049ee7803e0f98c, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:58:42,314 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731751108989.7c7f87a6124bfe2f4049ee7803e0f98c., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c7f87a6124bfe2f4049ee7803e0f98c, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:58:42,314 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731751108989.7c7f87a6124bfe2f4049ee7803e0f98c., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 because the exception is null or not the one we care about 2024-11-16T09:58:42,315 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c7f87a6124bfe2f4049ee7803e0f98c, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T09:58:42,315 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52392 deadline: 1731751182313, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c7f87a6124bfe2f4049ee7803e0f98c, server=037c58a19ddb,37267,1731751077844 2024-11-16T09:58:42,315 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731751108989.7c7f87a6124bfe2f4049ee7803e0f98c., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 , the old value is region=TestAcidGuarantees,,1731751108989.7c7f87a6124bfe2f4049ee7803e0f98c., hostname=037c58a19ddb,37267,1731751077844, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c7f87a6124bfe2f4049ee7803e0f98c, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:58:42,315 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731751108989.7c7f87a6124bfe2f4049ee7803e0f98c., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c7f87a6124bfe2f4049ee7803e0f98c, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:58:42,315 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731751108989.7c7f87a6124bfe2f4049ee7803e0f98c., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 because the exception is null or not the one we care about 2024-11-16T09:58:42,315 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731751108989.7c7f87a6124bfe2f4049ee7803e0f98c., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 , the old value is region=TestAcidGuarantees,,1731751108989.7c7f87a6124bfe2f4049ee7803e0f98c., hostname=037c58a19ddb,37267,1731751077844, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c7f87a6124bfe2f4049ee7803e0f98c, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:58:42,315 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731751108989.7c7f87a6124bfe2f4049ee7803e0f98c., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c7f87a6124bfe2f4049ee7803e0f98c, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:58:42,315 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731751108989.7c7f87a6124bfe2f4049ee7803e0f98c., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 because the exception is null or not the one we care about 2024-11-16T09:58:42,329 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37591 is added to blk_1073742021_1197 (size=12304) 2024-11-16T09:58:42,345 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c7f87a6124bfe2f4049ee7803e0f98c, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T09:58:42,345 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52428 deadline: 1731751182343, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c7f87a6124bfe2f4049ee7803e0f98c, server=037c58a19ddb,37267,1731751077844 2024-11-16T09:58:42,346 WARN [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRpcRetryingCaller(177): Call to 037c58a19ddb,37267,1731751077844 for 'test_row_1' in 7c7f87a6124bfe2f4049ee7803e0f98c of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4190 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c7f87a6124bfe2f4049ee7803e0f98c, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor44.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c7f87a6124bfe2f4049ee7803e0f98c, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-16T09:58:42,347 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731751108989.7c7f87a6124bfe2f4049ee7803e0f98c., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 , the old value is region=TestAcidGuarantees,,1731751108989.7c7f87a6124bfe2f4049ee7803e0f98c., hostname=037c58a19ddb,37267,1731751077844, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c7f87a6124bfe2f4049ee7803e0f98c, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:58:42,347 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731751108989.7c7f87a6124bfe2f4049ee7803e0f98c., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c7f87a6124bfe2f4049ee7803e0f98c, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:58:42,347 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731751108989.7c7f87a6124bfe2f4049ee7803e0f98c., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 because the exception is null or not the one we care about 2024-11-16T09:58:42,399 INFO [master/037c58a19ddb:0.Chore.1 {}] migrate.RollingUpgradeChore(116): There is no table to migrate StoreFileTracker! 2024-11-16T09:58:42,399 INFO [master/037c58a19ddb:0.Chore.1 {}] migrate.RollingUpgradeChore(85): All Rolling-Upgrade tasks are complete, shutdown RollingUpgradeChore! 2024-11-16T09:58:42,424 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c7f87a6124bfe2f4049ee7803e0f98c, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T09:58:42,424 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] ipc.CallRunner(138): callId: 78 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52400 deadline: 1731751182423, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c7f87a6124bfe2f4049ee7803e0f98c, server=037c58a19ddb,37267,1731751077844 2024-11-16T09:58:42,425 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731751108989.7c7f87a6124bfe2f4049ee7803e0f98c., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 , the old value is region=TestAcidGuarantees,,1731751108989.7c7f87a6124bfe2f4049ee7803e0f98c., hostname=037c58a19ddb,37267,1731751077844, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c7f87a6124bfe2f4049ee7803e0f98c, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:58:42,425 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731751108989.7c7f87a6124bfe2f4049ee7803e0f98c., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c7f87a6124bfe2f4049ee7803e0f98c, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:58:42,425 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731751108989.7c7f87a6124bfe2f4049ee7803e0f98c., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 because the exception is null or not the one we care about 2024-11-16T09:58:42,425 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c7f87a6124bfe2f4049ee7803e0f98c, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T09:58:42,425 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] ipc.CallRunner(138): callId: 77 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52396 deadline: 1731751182423, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c7f87a6124bfe2f4049ee7803e0f98c, server=037c58a19ddb,37267,1731751077844 2024-11-16T09:58:42,426 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c7f87a6124bfe2f4049ee7803e0f98c, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T09:58:42,426 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731751108989.7c7f87a6124bfe2f4049ee7803e0f98c., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 , the old value is region=TestAcidGuarantees,,1731751108989.7c7f87a6124bfe2f4049ee7803e0f98c., hostname=037c58a19ddb,37267,1731751077844, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c7f87a6124bfe2f4049ee7803e0f98c, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:58:42,426 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] ipc.CallRunner(138): callId: 74 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52420 deadline: 1731751182424, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c7f87a6124bfe2f4049ee7803e0f98c, server=037c58a19ddb,37267,1731751077844 2024-11-16T09:58:42,426 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731751108989.7c7f87a6124bfe2f4049ee7803e0f98c., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c7f87a6124bfe2f4049ee7803e0f98c, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:58:42,426 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731751108989.7c7f87a6124bfe2f4049ee7803e0f98c., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 because the exception is null or not the one we care about 2024-11-16T09:58:42,426 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c7f87a6124bfe2f4049ee7803e0f98c, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T09:58:42,426 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52392 deadline: 1731751182424, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c7f87a6124bfe2f4049ee7803e0f98c, server=037c58a19ddb,37267,1731751077844 2024-11-16T09:58:42,426 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731751108989.7c7f87a6124bfe2f4049ee7803e0f98c., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 , the old value is region=TestAcidGuarantees,,1731751108989.7c7f87a6124bfe2f4049ee7803e0f98c., hostname=037c58a19ddb,37267,1731751077844, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c7f87a6124bfe2f4049ee7803e0f98c, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:58:42,426 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731751108989.7c7f87a6124bfe2f4049ee7803e0f98c., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c7f87a6124bfe2f4049ee7803e0f98c, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:58:42,426 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731751108989.7c7f87a6124bfe2f4049ee7803e0f98c., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 because the exception is null or not the one we care about 2024-11-16T09:58:42,427 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731751108989.7c7f87a6124bfe2f4049ee7803e0f98c., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 , the old value is region=TestAcidGuarantees,,1731751108989.7c7f87a6124bfe2f4049ee7803e0f98c., hostname=037c58a19ddb,37267,1731751077844, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c7f87a6124bfe2f4049ee7803e0f98c, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:58:42,427 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731751108989.7c7f87a6124bfe2f4049ee7803e0f98c., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c7f87a6124bfe2f4049ee7803e0f98c, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:58:42,427 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731751108989.7c7f87a6124bfe2f4049ee7803e0f98c., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 because the exception is null or not the one we care about 2024-11-16T09:58:42,635 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c7f87a6124bfe2f4049ee7803e0f98c, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T09:58:42,635 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] ipc.CallRunner(138): callId: 78 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52396 deadline: 1731751182633, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c7f87a6124bfe2f4049ee7803e0f98c, server=037c58a19ddb,37267,1731751077844 2024-11-16T09:58:42,635 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c7f87a6124bfe2f4049ee7803e0f98c, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T09:58:42,636 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52392 deadline: 1731751182633, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c7f87a6124bfe2f4049ee7803e0f98c, server=037c58a19ddb,37267,1731751077844 2024-11-16T09:58:42,636 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c7f87a6124bfe2f4049ee7803e0f98c, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T09:58:42,636 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] ipc.CallRunner(138): callId: 75 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52420 deadline: 1731751182633, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c7f87a6124bfe2f4049ee7803e0f98c, server=037c58a19ddb,37267,1731751077844 2024-11-16T09:58:42,636 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c7f87a6124bfe2f4049ee7803e0f98c, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T09:58:42,636 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] ipc.CallRunner(138): callId: 79 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52400 deadline: 1731751182633, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c7f87a6124bfe2f4049ee7803e0f98c, server=037c58a19ddb,37267,1731751077844 2024-11-16T09:58:42,637 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731751108989.7c7f87a6124bfe2f4049ee7803e0f98c., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 , the old value is region=TestAcidGuarantees,,1731751108989.7c7f87a6124bfe2f4049ee7803e0f98c., hostname=037c58a19ddb,37267,1731751077844, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c7f87a6124bfe2f4049ee7803e0f98c, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:58:42,637 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731751108989.7c7f87a6124bfe2f4049ee7803e0f98c., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c7f87a6124bfe2f4049ee7803e0f98c, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:58:42,637 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731751108989.7c7f87a6124bfe2f4049ee7803e0f98c., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 because the exception is null or not the one we care about 2024-11-16T09:58:42,637 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731751108989.7c7f87a6124bfe2f4049ee7803e0f98c., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 , the old value is region=TestAcidGuarantees,,1731751108989.7c7f87a6124bfe2f4049ee7803e0f98c., hostname=037c58a19ddb,37267,1731751077844, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c7f87a6124bfe2f4049ee7803e0f98c, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:58:42,637 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731751108989.7c7f87a6124bfe2f4049ee7803e0f98c., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c7f87a6124bfe2f4049ee7803e0f98c, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:58:42,637 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731751108989.7c7f87a6124bfe2f4049ee7803e0f98c., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 because the exception is null or not the one we care about 2024-11-16T09:58:42,638 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731751108989.7c7f87a6124bfe2f4049ee7803e0f98c., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 , the old value is region=TestAcidGuarantees,,1731751108989.7c7f87a6124bfe2f4049ee7803e0f98c., hostname=037c58a19ddb,37267,1731751077844, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c7f87a6124bfe2f4049ee7803e0f98c, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:58:42,638 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731751108989.7c7f87a6124bfe2f4049ee7803e0f98c., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c7f87a6124bfe2f4049ee7803e0f98c, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:58:42,638 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731751108989.7c7f87a6124bfe2f4049ee7803e0f98c., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 because the exception is null or not the one we care about 2024-11-16T09:58:42,638 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731751108989.7c7f87a6124bfe2f4049ee7803e0f98c., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 , the old value is region=TestAcidGuarantees,,1731751108989.7c7f87a6124bfe2f4049ee7803e0f98c., hostname=037c58a19ddb,37267,1731751077844, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c7f87a6124bfe2f4049ee7803e0f98c, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:58:42,638 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731751108989.7c7f87a6124bfe2f4049ee7803e0f98c., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c7f87a6124bfe2f4049ee7803e0f98c, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:58:42,638 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731751108989.7c7f87a6124bfe2f4049ee7803e0f98c., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 because the exception is null or not the one we care about 2024-11-16T09:58:42,642 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37877 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=42 2024-11-16T09:58:42,643 INFO [RPCClient-NioEventLoopGroup-4-7 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-16T09:58:42,643 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37877 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-16T09:58:42,645 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37877 {}] procedure2.ProcedureExecutor(1139): Stored pid=44, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=44, table=TestAcidGuarantees 2024-11-16T09:58:42,646 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37877 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=44 2024-11-16T09:58:42,646 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=44, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=44, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-16T09:58:42,647 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=44, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=44, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-16T09:58:42,647 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=45, ppid=44, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-16T09:58:42,732 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T09:58:42,738 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111650618aa9f5c94860bb796babe0a66499_7c7f87a6124bfe2f4049ee7803e0f98c to hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111650618aa9f5c94860bb796babe0a66499_7c7f87a6124bfe2f4049ee7803e0f98c 2024-11-16T09:58:42,739 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/7c7f87a6124bfe2f4049ee7803e0f98c/.tmp/A/9db025a49e5b45598af87c7bf658d34b, store: [table=TestAcidGuarantees family=A region=7c7f87a6124bfe2f4049ee7803e0f98c] 2024-11-16T09:58:42,740 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/7c7f87a6124bfe2f4049ee7803e0f98c/.tmp/A/9db025a49e5b45598af87c7bf658d34b is 175, key is test_row_0/A:col10/1731751121633/Put/seqid=0 2024-11-16T09:58:42,753 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37877 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=44 2024-11-16T09:58:42,753 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37591 is added to blk_1073742022_1198 (size=31105) 2024-11-16T09:58:42,760 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=213, memsize=49.2 K, hasBloomFilter=true, into tmp file hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/7c7f87a6124bfe2f4049ee7803e0f98c/.tmp/A/9db025a49e5b45598af87c7bf658d34b 2024-11-16T09:58:42,777 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/7c7f87a6124bfe2f4049ee7803e0f98c/.tmp/B/0390cee6611b4f5b8512cca4042a155d is 50, key is test_row_0/B:col10/1731751121633/Put/seqid=0 2024-11-16T09:58:42,790 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37591 is added to blk_1073742023_1199 (size=12151) 2024-11-16T09:58:42,798 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=213 (bloomFilter=true), to=hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/7c7f87a6124bfe2f4049ee7803e0f98c/.tmp/B/0390cee6611b4f5b8512cca4042a155d 2024-11-16T09:58:42,800 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=37267 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=45 2024-11-16T09:58:42,800 DEBUG [RS_FLUSH_OPERATIONS-regionserver/037c58a19ddb:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731751108989.7c7f87a6124bfe2f4049ee7803e0f98c. 2024-11-16T09:58:42,800 DEBUG [RS_FLUSH_OPERATIONS-regionserver/037c58a19ddb:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731751108989.7c7f87a6124bfe2f4049ee7803e0f98c. as already flushing 2024-11-16T09:58:42,800 DEBUG [RS_FLUSH_OPERATIONS-regionserver/037c58a19ddb:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731751108989.7c7f87a6124bfe2f4049ee7803e0f98c. 2024-11-16T09:58:42,800 ERROR [RS_FLUSH_OPERATIONS-regionserver/037c58a19ddb:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] handler.RSProcedureHandler(58): pid=45 java.io.IOException: Unable to complete flush {ENCODED => 7c7f87a6124bfe2f4049ee7803e0f98c, NAME => 'TestAcidGuarantees,,1731751108989.7c7f87a6124bfe2f4049ee7803e0f98c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T09:58:42,800 DEBUG [RS_FLUSH_OPERATIONS-regionserver/037c58a19ddb:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=45 java.io.IOException: Unable to complete flush {ENCODED => 7c7f87a6124bfe2f4049ee7803e0f98c, NAME => 'TestAcidGuarantees,,1731751108989.7c7f87a6124bfe2f4049ee7803e0f98c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T09:58:42,801 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37877 {}] master.HMaster(4177): Remote procedure failed, pid=45 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 7c7f87a6124bfe2f4049ee7803e0f98c, NAME => 'TestAcidGuarantees,,1731751108989.7c7f87a6124bfe2f4049ee7803e0f98c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 7c7f87a6124bfe2f4049ee7803e0f98c, NAME => 'TestAcidGuarantees,,1731751108989.7c7f87a6124bfe2f4049ee7803e0f98c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T09:58:42,815 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/7c7f87a6124bfe2f4049ee7803e0f98c/.tmp/C/a0721c8cf1244143b39fcb3fb61616b6 is 50, key is test_row_0/C:col10/1731751121633/Put/seqid=0 2024-11-16T09:58:42,833 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37591 is added to blk_1073742024_1200 (size=12151) 2024-11-16T09:58:42,834 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=213 (bloomFilter=true), to=hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/7c7f87a6124bfe2f4049ee7803e0f98c/.tmp/C/a0721c8cf1244143b39fcb3fb61616b6 2024-11-16T09:58:42,839 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/7c7f87a6124bfe2f4049ee7803e0f98c/.tmp/A/9db025a49e5b45598af87c7bf658d34b as hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/7c7f87a6124bfe2f4049ee7803e0f98c/A/9db025a49e5b45598af87c7bf658d34b 2024-11-16T09:58:42,847 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/7c7f87a6124bfe2f4049ee7803e0f98c/A/9db025a49e5b45598af87c7bf658d34b, entries=150, sequenceid=213, filesize=30.4 K 2024-11-16T09:58:42,848 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/7c7f87a6124bfe2f4049ee7803e0f98c/.tmp/B/0390cee6611b4f5b8512cca4042a155d as hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/7c7f87a6124bfe2f4049ee7803e0f98c/B/0390cee6611b4f5b8512cca4042a155d 2024-11-16T09:58:42,858 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/7c7f87a6124bfe2f4049ee7803e0f98c/B/0390cee6611b4f5b8512cca4042a155d, entries=150, sequenceid=213, filesize=11.9 K 2024-11-16T09:58:42,862 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/7c7f87a6124bfe2f4049ee7803e0f98c/.tmp/C/a0721c8cf1244143b39fcb3fb61616b6 as hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/7c7f87a6124bfe2f4049ee7803e0f98c/C/a0721c8cf1244143b39fcb3fb61616b6 2024-11-16T09:58:42,873 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/7c7f87a6124bfe2f4049ee7803e0f98c/C/a0721c8cf1244143b39fcb3fb61616b6, entries=150, sequenceid=213, filesize=11.9 K 2024-11-16T09:58:42,874 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~147.60 KB/151140, heapSize ~387.42 KB/396720, currentSize=60.38 KB/61830 for 7c7f87a6124bfe2f4049ee7803e0f98c in 588ms, sequenceid=213, compaction requested=true 2024-11-16T09:58:42,874 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 7c7f87a6124bfe2f4049ee7803e0f98c: 2024-11-16T09:58:42,874 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 7c7f87a6124bfe2f4049ee7803e0f98c:A, priority=-2147483648, current under compaction store size is 1 2024-11-16T09:58:42,874 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-16T09:58:42,875 DEBUG [RS:0;037c58a19ddb:37267-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-16T09:58:42,875 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 7c7f87a6124bfe2f4049ee7803e0f98c:B, priority=-2147483648, current under compaction store size is 2 2024-11-16T09:58:42,875 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-16T09:58:42,875 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 7c7f87a6124bfe2f4049ee7803e0f98c:C, priority=-2147483648, current under compaction store size is 3 2024-11-16T09:58:42,875 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=2), splitQueue=0 2024-11-16T09:58:42,877 DEBUG [RS:0;037c58a19ddb:37267-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-16T09:58:42,879 DEBUG [RS:0;037c58a19ddb:37267-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 119619 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-16T09:58:42,879 DEBUG [RS:0;037c58a19ddb:37267-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36829 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-16T09:58:42,879 DEBUG [RS:0;037c58a19ddb:37267-longCompactions-0 {}] regionserver.HStore(1541): 7c7f87a6124bfe2f4049ee7803e0f98c/A is initiating minor compaction (all files) 2024-11-16T09:58:42,879 DEBUG [RS:0;037c58a19ddb:37267-shortCompactions-0 {}] regionserver.HStore(1541): 7c7f87a6124bfe2f4049ee7803e0f98c/B is initiating minor compaction (all files) 2024-11-16T09:58:42,879 INFO [RS:0;037c58a19ddb:37267-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 7c7f87a6124bfe2f4049ee7803e0f98c/A in TestAcidGuarantees,,1731751108989.7c7f87a6124bfe2f4049ee7803e0f98c. 2024-11-16T09:58:42,879 INFO [RS:0;037c58a19ddb:37267-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 7c7f87a6124bfe2f4049ee7803e0f98c/B in TestAcidGuarantees,,1731751108989.7c7f87a6124bfe2f4049ee7803e0f98c. 2024-11-16T09:58:42,879 INFO [RS:0;037c58a19ddb:37267-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/7c7f87a6124bfe2f4049ee7803e0f98c/A/3bff852129eb44848826f920130a1069, hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/7c7f87a6124bfe2f4049ee7803e0f98c/A/42e604f249434caf82bcc0dc3532e984, hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/7c7f87a6124bfe2f4049ee7803e0f98c/A/9db025a49e5b45598af87c7bf658d34b] into tmpdir=hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/7c7f87a6124bfe2f4049ee7803e0f98c/.tmp, totalSize=116.8 K 2024-11-16T09:58:42,879 INFO [RS:0;037c58a19ddb:37267-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/7c7f87a6124bfe2f4049ee7803e0f98c/B/2bf5259150774f07a21de5b3fb39be24, hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/7c7f87a6124bfe2f4049ee7803e0f98c/B/bc151e55d37549828dba2a7feccb86eb, hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/7c7f87a6124bfe2f4049ee7803e0f98c/B/0390cee6611b4f5b8512cca4042a155d] into tmpdir=hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/7c7f87a6124bfe2f4049ee7803e0f98c/.tmp, totalSize=36.0 K 2024-11-16T09:58:42,879 INFO [RS:0;037c58a19ddb:37267-longCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731751108989.7c7f87a6124bfe2f4049ee7803e0f98c. 2024-11-16T09:58:42,879 DEBUG [RS:0;037c58a19ddb:37267-longCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731751108989.7c7f87a6124bfe2f4049ee7803e0f98c. files: [hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/7c7f87a6124bfe2f4049ee7803e0f98c/A/3bff852129eb44848826f920130a1069, hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/7c7f87a6124bfe2f4049ee7803e0f98c/A/42e604f249434caf82bcc0dc3532e984, hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/7c7f87a6124bfe2f4049ee7803e0f98c/A/9db025a49e5b45598af87c7bf658d34b] 2024-11-16T09:58:42,880 DEBUG [RS:0;037c58a19ddb:37267-shortCompactions-0 {}] compactions.Compactor(225): Compacting 2bf5259150774f07a21de5b3fb39be24, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=173, earliestPutTs=1731751119982 2024-11-16T09:58:42,880 DEBUG [RS:0;037c58a19ddb:37267-longCompactions-0 {}] compactions.Compactor(225): Compacting 3bff852129eb44848826f920130a1069, keycount=150, bloomtype=ROW, size=30.7 K, encoding=NONE, compression=NONE, seqNum=173, earliestPutTs=1731751119982 2024-11-16T09:58:42,880 DEBUG [RS:0;037c58a19ddb:37267-longCompactions-0 {}] compactions.Compactor(225): Compacting 42e604f249434caf82bcc0dc3532e984, keycount=300, bloomtype=ROW, size=55.7 K, encoding=NONE, compression=NONE, seqNum=187, earliestPutTs=1731751121240 2024-11-16T09:58:42,880 DEBUG [RS:0;037c58a19ddb:37267-shortCompactions-0 {}] compactions.Compactor(225): Compacting bc151e55d37549828dba2a7feccb86eb, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=187, earliestPutTs=1731751121574 2024-11-16T09:58:42,880 DEBUG [RS:0;037c58a19ddb:37267-longCompactions-0 {}] compactions.Compactor(225): Compacting 9db025a49e5b45598af87c7bf658d34b, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=213, earliestPutTs=1731751121623 2024-11-16T09:58:42,881 DEBUG [RS:0;037c58a19ddb:37267-shortCompactions-0 {}] compactions.Compactor(225): Compacting 0390cee6611b4f5b8512cca4042a155d, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=213, earliestPutTs=1731751121623 2024-11-16T09:58:42,893 INFO [RS:0;037c58a19ddb:37267-longCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=7c7f87a6124bfe2f4049ee7803e0f98c] 2024-11-16T09:58:42,904 DEBUG [RS:0;037c58a19ddb:37267-longCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241116ff16f2e1a2c8402791c5629e9a9fa68c_7c7f87a6124bfe2f4049ee7803e0f98c store=[table=TestAcidGuarantees family=A region=7c7f87a6124bfe2f4049ee7803e0f98c] 2024-11-16T09:58:42,906 INFO [RS:0;037c58a19ddb:37267-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 7c7f87a6124bfe2f4049ee7803e0f98c#B#compaction#172 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-16T09:58:42,906 DEBUG [RS:0;037c58a19ddb:37267-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/7c7f87a6124bfe2f4049ee7803e0f98c/.tmp/B/9989e7c7af994090af37ab514c81a8d1 is 50, key is test_row_0/B:col10/1731751121633/Put/seqid=0 2024-11-16T09:58:42,907 DEBUG [RS:0;037c58a19ddb:37267-longCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241116ff16f2e1a2c8402791c5629e9a9fa68c_7c7f87a6124bfe2f4049ee7803e0f98c, store=[table=TestAcidGuarantees family=A region=7c7f87a6124bfe2f4049ee7803e0f98c] 2024-11-16T09:58:42,907 DEBUG [RS:0;037c58a19ddb:37267-longCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241116ff16f2e1a2c8402791c5629e9a9fa68c_7c7f87a6124bfe2f4049ee7803e0f98c because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=7c7f87a6124bfe2f4049ee7803e0f98c] 2024-11-16T09:58:42,916 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37591 is added to blk_1073742025_1201 (size=12629) 2024-11-16T09:58:42,925 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37591 is added to blk_1073742026_1202 (size=4469) 2024-11-16T09:58:42,945 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] regionserver.HRegion(8855): Flush requested on 7c7f87a6124bfe2f4049ee7803e0f98c 2024-11-16T09:58:42,945 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-16T09:58:42,945 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 7c7f87a6124bfe2f4049ee7803e0f98c 3/3 column families, dataSize=67.09 KB heapSize=176.53 KB 2024-11-16T09:58:42,953 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=37267 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=45 2024-11-16T09:58:42,953 DEBUG [RS_FLUSH_OPERATIONS-regionserver/037c58a19ddb:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731751108989.7c7f87a6124bfe2f4049ee7803e0f98c. 2024-11-16T09:58:42,954 DEBUG [RS_FLUSH_OPERATIONS-regionserver/037c58a19ddb:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731751108989.7c7f87a6124bfe2f4049ee7803e0f98c. as already flushing 2024-11-16T09:58:42,954 DEBUG [RS_FLUSH_OPERATIONS-regionserver/037c58a19ddb:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731751108989.7c7f87a6124bfe2f4049ee7803e0f98c. 2024-11-16T09:58:42,954 ERROR [RS_FLUSH_OPERATIONS-regionserver/037c58a19ddb:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] handler.RSProcedureHandler(58): pid=45 java.io.IOException: Unable to complete flush {ENCODED => 7c7f87a6124bfe2f4049ee7803e0f98c, NAME => 'TestAcidGuarantees,,1731751108989.7c7f87a6124bfe2f4049ee7803e0f98c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T09:58:42,954 DEBUG [RS_FLUSH_OPERATIONS-regionserver/037c58a19ddb:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=45 java.io.IOException: Unable to complete flush {ENCODED => 7c7f87a6124bfe2f4049ee7803e0f98c, NAME => 'TestAcidGuarantees,,1731751108989.7c7f87a6124bfe2f4049ee7803e0f98c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T09:58:42,955 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37877 {}] master.HMaster(4177): Remote procedure failed, pid=45 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 7c7f87a6124bfe2f4049ee7803e0f98c, NAME => 'TestAcidGuarantees,,1731751108989.7c7f87a6124bfe2f4049ee7803e0f98c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 7c7f87a6124bfe2f4049ee7803e0f98c, NAME => 'TestAcidGuarantees,,1731751108989.7c7f87a6124bfe2f4049ee7803e0f98c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T09:58:42,955 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241116393b75585278420e926a6eb898d59507_7c7f87a6124bfe2f4049ee7803e0f98c is 50, key is test_row_0/A:col10/1731751122945/Put/seqid=0 2024-11-16T09:58:42,962 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37591 is added to blk_1073742027_1203 (size=14794) 2024-11-16T09:58:42,963 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37877 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=44 2024-11-16T09:58:42,970 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T09:58:42,977 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241116393b75585278420e926a6eb898d59507_7c7f87a6124bfe2f4049ee7803e0f98c to hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241116393b75585278420e926a6eb898d59507_7c7f87a6124bfe2f4049ee7803e0f98c 2024-11-16T09:58:42,978 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/7c7f87a6124bfe2f4049ee7803e0f98c/.tmp/A/2049d15304474f38af50e9ff363d240f, store: [table=TestAcidGuarantees family=A region=7c7f87a6124bfe2f4049ee7803e0f98c] 2024-11-16T09:58:42,979 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/7c7f87a6124bfe2f4049ee7803e0f98c/.tmp/A/2049d15304474f38af50e9ff363d240f is 175, key is test_row_0/A:col10/1731751122945/Put/seqid=0 2024-11-16T09:58:42,983 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37591 is added to blk_1073742028_1204 (size=39749) 2024-11-16T09:58:43,017 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c7f87a6124bfe2f4049ee7803e0f98c, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T09:58:43,017 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] ipc.CallRunner(138): callId: 86 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52400 deadline: 1731751183016, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c7f87a6124bfe2f4049ee7803e0f98c, server=037c58a19ddb,37267,1731751077844 2024-11-16T09:58:43,018 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731751108989.7c7f87a6124bfe2f4049ee7803e0f98c., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 , the old value is region=TestAcidGuarantees,,1731751108989.7c7f87a6124bfe2f4049ee7803e0f98c., hostname=037c58a19ddb,37267,1731751077844, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c7f87a6124bfe2f4049ee7803e0f98c, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:58:43,018 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731751108989.7c7f87a6124bfe2f4049ee7803e0f98c., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c7f87a6124bfe2f4049ee7803e0f98c, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:58:43,018 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731751108989.7c7f87a6124bfe2f4049ee7803e0f98c., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 because the exception is null or not the one we care about 2024-11-16T09:58:43,020 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c7f87a6124bfe2f4049ee7803e0f98c, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T09:58:43,021 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] ipc.CallRunner(138): callId: 84 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52396 deadline: 1731751183016, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c7f87a6124bfe2f4049ee7803e0f98c, server=037c58a19ddb,37267,1731751077844 2024-11-16T09:58:43,021 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731751108989.7c7f87a6124bfe2f4049ee7803e0f98c., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 , the old value is region=TestAcidGuarantees,,1731751108989.7c7f87a6124bfe2f4049ee7803e0f98c., hostname=037c58a19ddb,37267,1731751077844, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c7f87a6124bfe2f4049ee7803e0f98c, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:58:43,021 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c7f87a6124bfe2f4049ee7803e0f98c, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T09:58:43,021 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731751108989.7c7f87a6124bfe2f4049ee7803e0f98c., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c7f87a6124bfe2f4049ee7803e0f98c, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:58:43,021 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c7f87a6124bfe2f4049ee7803e0f98c, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T09:58:43,021 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731751108989.7c7f87a6124bfe2f4049ee7803e0f98c., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 because the exception is null or not the one we care about 2024-11-16T09:58:43,021 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52392 deadline: 1731751183017, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c7f87a6124bfe2f4049ee7803e0f98c, server=037c58a19ddb,37267,1731751077844 2024-11-16T09:58:43,021 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] ipc.CallRunner(138): callId: 81 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52420 deadline: 1731751183017, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c7f87a6124bfe2f4049ee7803e0f98c, server=037c58a19ddb,37267,1731751077844 2024-11-16T09:58:43,021 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731751108989.7c7f87a6124bfe2f4049ee7803e0f98c., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 , the old value is region=TestAcidGuarantees,,1731751108989.7c7f87a6124bfe2f4049ee7803e0f98c., hostname=037c58a19ddb,37267,1731751077844, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c7f87a6124bfe2f4049ee7803e0f98c, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:58:43,021 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731751108989.7c7f87a6124bfe2f4049ee7803e0f98c., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c7f87a6124bfe2f4049ee7803e0f98c, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:58:43,021 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731751108989.7c7f87a6124bfe2f4049ee7803e0f98c., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 , the old value is region=TestAcidGuarantees,,1731751108989.7c7f87a6124bfe2f4049ee7803e0f98c., hostname=037c58a19ddb,37267,1731751077844, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c7f87a6124bfe2f4049ee7803e0f98c, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:58:43,021 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731751108989.7c7f87a6124bfe2f4049ee7803e0f98c., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 because the exception is null or not the one we care about 2024-11-16T09:58:43,021 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731751108989.7c7f87a6124bfe2f4049ee7803e0f98c., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c7f87a6124bfe2f4049ee7803e0f98c, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:58:43,021 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731751108989.7c7f87a6124bfe2f4049ee7803e0f98c., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 because the exception is null or not the one we care about 2024-11-16T09:58:43,107 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=37267 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=45 2024-11-16T09:58:43,107 DEBUG [RS_FLUSH_OPERATIONS-regionserver/037c58a19ddb:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731751108989.7c7f87a6124bfe2f4049ee7803e0f98c. 2024-11-16T09:58:43,107 DEBUG [RS_FLUSH_OPERATIONS-regionserver/037c58a19ddb:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731751108989.7c7f87a6124bfe2f4049ee7803e0f98c. as already flushing 2024-11-16T09:58:43,107 DEBUG [RS_FLUSH_OPERATIONS-regionserver/037c58a19ddb:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731751108989.7c7f87a6124bfe2f4049ee7803e0f98c. 2024-11-16T09:58:43,107 ERROR [RS_FLUSH_OPERATIONS-regionserver/037c58a19ddb:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] handler.RSProcedureHandler(58): pid=45 java.io.IOException: Unable to complete flush {ENCODED => 7c7f87a6124bfe2f4049ee7803e0f98c, NAME => 'TestAcidGuarantees,,1731751108989.7c7f87a6124bfe2f4049ee7803e0f98c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T09:58:43,107 DEBUG [RS_FLUSH_OPERATIONS-regionserver/037c58a19ddb:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=45 java.io.IOException: Unable to complete flush {ENCODED => 7c7f87a6124bfe2f4049ee7803e0f98c, NAME => 'TestAcidGuarantees,,1731751108989.7c7f87a6124bfe2f4049ee7803e0f98c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T09:58:43,108 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37877 {}] master.HMaster(4177): Remote procedure failed, pid=45 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 7c7f87a6124bfe2f4049ee7803e0f98c, NAME => 'TestAcidGuarantees,,1731751108989.7c7f87a6124bfe2f4049ee7803e0f98c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 7c7f87a6124bfe2f4049ee7803e0f98c, NAME => 'TestAcidGuarantees,,1731751108989.7c7f87a6124bfe2f4049ee7803e0f98c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T09:58:43,125 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c7f87a6124bfe2f4049ee7803e0f98c, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T09:58:43,125 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] ipc.CallRunner(138): callId: 87 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52400 deadline: 1731751183123, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c7f87a6124bfe2f4049ee7803e0f98c, server=037c58a19ddb,37267,1731751077844 2024-11-16T09:58:43,126 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731751108989.7c7f87a6124bfe2f4049ee7803e0f98c., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 , the old value is region=TestAcidGuarantees,,1731751108989.7c7f87a6124bfe2f4049ee7803e0f98c., hostname=037c58a19ddb,37267,1731751077844, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c7f87a6124bfe2f4049ee7803e0f98c, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:58:43,126 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731751108989.7c7f87a6124bfe2f4049ee7803e0f98c., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c7f87a6124bfe2f4049ee7803e0f98c, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:58:43,126 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c7f87a6124bfe2f4049ee7803e0f98c, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T09:58:43,126 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731751108989.7c7f87a6124bfe2f4049ee7803e0f98c., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 because the exception is null or not the one we care about 2024-11-16T09:58:43,126 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] ipc.CallRunner(138): callId: 85 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52396 deadline: 1731751183123, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c7f87a6124bfe2f4049ee7803e0f98c, server=037c58a19ddb,37267,1731751077844 2024-11-16T09:58:43,126 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731751108989.7c7f87a6124bfe2f4049ee7803e0f98c., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 , the old value is region=TestAcidGuarantees,,1731751108989.7c7f87a6124bfe2f4049ee7803e0f98c., hostname=037c58a19ddb,37267,1731751077844, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c7f87a6124bfe2f4049ee7803e0f98c, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:58:43,126 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731751108989.7c7f87a6124bfe2f4049ee7803e0f98c., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c7f87a6124bfe2f4049ee7803e0f98c, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:58:43,126 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731751108989.7c7f87a6124bfe2f4049ee7803e0f98c., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 because the exception is null or not the one we care about 2024-11-16T09:58:43,138 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c7f87a6124bfe2f4049ee7803e0f98c, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T09:58:43,139 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c7f87a6124bfe2f4049ee7803e0f98c, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T09:58:43,139 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] ipc.CallRunner(138): callId: 71 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52392 deadline: 1731751183132, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c7f87a6124bfe2f4049ee7803e0f98c, server=037c58a19ddb,37267,1731751077844 2024-11-16T09:58:43,139 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] ipc.CallRunner(138): callId: 82 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52420 deadline: 1731751183133, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c7f87a6124bfe2f4049ee7803e0f98c, server=037c58a19ddb,37267,1731751077844 2024-11-16T09:58:43,139 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731751108989.7c7f87a6124bfe2f4049ee7803e0f98c., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 , the old value is region=TestAcidGuarantees,,1731751108989.7c7f87a6124bfe2f4049ee7803e0f98c., hostname=037c58a19ddb,37267,1731751077844, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c7f87a6124bfe2f4049ee7803e0f98c, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:58:43,139 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731751108989.7c7f87a6124bfe2f4049ee7803e0f98c., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c7f87a6124bfe2f4049ee7803e0f98c, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:58:43,139 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731751108989.7c7f87a6124bfe2f4049ee7803e0f98c., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 because the exception is null or not the one we care about 2024-11-16T09:58:43,139 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731751108989.7c7f87a6124bfe2f4049ee7803e0f98c., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 , the old value is region=TestAcidGuarantees,,1731751108989.7c7f87a6124bfe2f4049ee7803e0f98c., hostname=037c58a19ddb,37267,1731751077844, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c7f87a6124bfe2f4049ee7803e0f98c, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:58:43,139 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731751108989.7c7f87a6124bfe2f4049ee7803e0f98c., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c7f87a6124bfe2f4049ee7803e0f98c, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:58:43,139 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731751108989.7c7f87a6124bfe2f4049ee7803e0f98c., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 because the exception is null or not the one we care about 2024-11-16T09:58:43,260 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=37267 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=45 2024-11-16T09:58:43,261 DEBUG [RS_FLUSH_OPERATIONS-regionserver/037c58a19ddb:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731751108989.7c7f87a6124bfe2f4049ee7803e0f98c. 2024-11-16T09:58:43,261 DEBUG [RS_FLUSH_OPERATIONS-regionserver/037c58a19ddb:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731751108989.7c7f87a6124bfe2f4049ee7803e0f98c. as already flushing 2024-11-16T09:58:43,261 DEBUG [RS_FLUSH_OPERATIONS-regionserver/037c58a19ddb:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731751108989.7c7f87a6124bfe2f4049ee7803e0f98c. 2024-11-16T09:58:43,261 ERROR [RS_FLUSH_OPERATIONS-regionserver/037c58a19ddb:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] handler.RSProcedureHandler(58): pid=45 java.io.IOException: Unable to complete flush {ENCODED => 7c7f87a6124bfe2f4049ee7803e0f98c, NAME => 'TestAcidGuarantees,,1731751108989.7c7f87a6124bfe2f4049ee7803e0f98c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T09:58:43,261 DEBUG [RS_FLUSH_OPERATIONS-regionserver/037c58a19ddb:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=45 java.io.IOException: Unable to complete flush {ENCODED => 7c7f87a6124bfe2f4049ee7803e0f98c, NAME => 'TestAcidGuarantees,,1731751108989.7c7f87a6124bfe2f4049ee7803e0f98c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T09:58:43,262 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37877 {}] master.HMaster(4177): Remote procedure failed, pid=45 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 7c7f87a6124bfe2f4049ee7803e0f98c, NAME => 'TestAcidGuarantees,,1731751108989.7c7f87a6124bfe2f4049ee7803e0f98c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 7c7f87a6124bfe2f4049ee7803e0f98c, NAME => 'TestAcidGuarantees,,1731751108989.7c7f87a6124bfe2f4049ee7803e0f98c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T09:58:43,273 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37877 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=44 2024-11-16T09:58:43,326 DEBUG [RS:0;037c58a19ddb:37267-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/7c7f87a6124bfe2f4049ee7803e0f98c/.tmp/B/9989e7c7af994090af37ab514c81a8d1 as hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/7c7f87a6124bfe2f4049ee7803e0f98c/B/9989e7c7af994090af37ab514c81a8d1 2024-11-16T09:58:43,329 INFO [RS:0;037c58a19ddb:37267-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 7c7f87a6124bfe2f4049ee7803e0f98c#A#compaction#171 average throughput is 0.06 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-16T09:58:43,330 DEBUG [RS:0;037c58a19ddb:37267-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/7c7f87a6124bfe2f4049ee7803e0f98c/.tmp/A/6b3a4c4820f0410ba008ad14ccf8af07 is 175, key is test_row_0/A:col10/1731751121633/Put/seqid=0 2024-11-16T09:58:43,333 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c7f87a6124bfe2f4049ee7803e0f98c, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T09:58:43,333 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] ipc.CallRunner(138): callId: 88 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52400 deadline: 1731751183332, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c7f87a6124bfe2f4049ee7803e0f98c, server=037c58a19ddb,37267,1731751077844 2024-11-16T09:58:43,334 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731751108989.7c7f87a6124bfe2f4049ee7803e0f98c., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 , the old value is region=TestAcidGuarantees,,1731751108989.7c7f87a6124bfe2f4049ee7803e0f98c., hostname=037c58a19ddb,37267,1731751077844, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c7f87a6124bfe2f4049ee7803e0f98c, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:58:43,334 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731751108989.7c7f87a6124bfe2f4049ee7803e0f98c., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c7f87a6124bfe2f4049ee7803e0f98c, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:58:43,334 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731751108989.7c7f87a6124bfe2f4049ee7803e0f98c., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 because the exception is null or not the one we care about 2024-11-16T09:58:43,334 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c7f87a6124bfe2f4049ee7803e0f98c, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T09:58:43,334 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] ipc.CallRunner(138): callId: 86 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52396 deadline: 1731751183333, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c7f87a6124bfe2f4049ee7803e0f98c, server=037c58a19ddb,37267,1731751077844 2024-11-16T09:58:43,335 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731751108989.7c7f87a6124bfe2f4049ee7803e0f98c., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 , the old value is region=TestAcidGuarantees,,1731751108989.7c7f87a6124bfe2f4049ee7803e0f98c., hostname=037c58a19ddb,37267,1731751077844, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c7f87a6124bfe2f4049ee7803e0f98c, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:58:43,335 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731751108989.7c7f87a6124bfe2f4049ee7803e0f98c., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c7f87a6124bfe2f4049ee7803e0f98c, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:58:43,335 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731751108989.7c7f87a6124bfe2f4049ee7803e0f98c., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 because the exception is null or not the one we care about 2024-11-16T09:58:43,336 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37591 is added to blk_1073742029_1205 (size=31583) 2024-11-16T09:58:43,338 INFO [RS:0;037c58a19ddb:37267-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 7c7f87a6124bfe2f4049ee7803e0f98c/B of 7c7f87a6124bfe2f4049ee7803e0f98c into 9989e7c7af994090af37ab514c81a8d1(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-16T09:58:43,338 DEBUG [RS:0;037c58a19ddb:37267-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 7c7f87a6124bfe2f4049ee7803e0f98c: 2024-11-16T09:58:43,338 INFO [RS:0;037c58a19ddb:37267-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731751108989.7c7f87a6124bfe2f4049ee7803e0f98c., storeName=7c7f87a6124bfe2f4049ee7803e0f98c/B, priority=13, startTime=1731751122875; duration=0sec 2024-11-16T09:58:43,338 DEBUG [RS:0;037c58a19ddb:37267-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-16T09:58:43,338 DEBUG [RS:0;037c58a19ddb:37267-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 7c7f87a6124bfe2f4049ee7803e0f98c:B 2024-11-16T09:58:43,338 DEBUG [RS:0;037c58a19ddb:37267-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-16T09:58:43,341 DEBUG [RS:0;037c58a19ddb:37267-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36829 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-16T09:58:43,341 DEBUG [RS:0;037c58a19ddb:37267-shortCompactions-0 {}] regionserver.HStore(1541): 7c7f87a6124bfe2f4049ee7803e0f98c/C is initiating minor compaction (all files) 2024-11-16T09:58:43,341 INFO [RS:0;037c58a19ddb:37267-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 7c7f87a6124bfe2f4049ee7803e0f98c/C in TestAcidGuarantees,,1731751108989.7c7f87a6124bfe2f4049ee7803e0f98c. 2024-11-16T09:58:43,341 INFO [RS:0;037c58a19ddb:37267-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/7c7f87a6124bfe2f4049ee7803e0f98c/C/20dd8e2f925044debf6455fa91c2494b, hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/7c7f87a6124bfe2f4049ee7803e0f98c/C/43bc34e4ddad4cbc85fe7916dd7fde9a, hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/7c7f87a6124bfe2f4049ee7803e0f98c/C/a0721c8cf1244143b39fcb3fb61616b6] into tmpdir=hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/7c7f87a6124bfe2f4049ee7803e0f98c/.tmp, totalSize=36.0 K 2024-11-16T09:58:43,342 DEBUG [RS:0;037c58a19ddb:37267-shortCompactions-0 {}] compactions.Compactor(225): Compacting 20dd8e2f925044debf6455fa91c2494b, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=173, earliestPutTs=1731751119982 2024-11-16T09:58:43,342 DEBUG [RS:0;037c58a19ddb:37267-shortCompactions-0 {}] compactions.Compactor(225): Compacting 43bc34e4ddad4cbc85fe7916dd7fde9a, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=187, earliestPutTs=1731751121574 2024-11-16T09:58:43,342 DEBUG [RS:0;037c58a19ddb:37267-shortCompactions-0 {}] compactions.Compactor(225): Compacting a0721c8cf1244143b39fcb3fb61616b6, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=213, earliestPutTs=1731751121623 2024-11-16T09:58:43,346 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c7f87a6124bfe2f4049ee7803e0f98c, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T09:58:43,346 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] ipc.CallRunner(138): callId: 72 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52392 deadline: 1731751183343, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c7f87a6124bfe2f4049ee7803e0f98c, server=037c58a19ddb,37267,1731751077844 2024-11-16T09:58:43,346 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731751108989.7c7f87a6124bfe2f4049ee7803e0f98c., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 , the old value is region=TestAcidGuarantees,,1731751108989.7c7f87a6124bfe2f4049ee7803e0f98c., hostname=037c58a19ddb,37267,1731751077844, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c7f87a6124bfe2f4049ee7803e0f98c, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:58:43,346 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c7f87a6124bfe2f4049ee7803e0f98c, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T09:58:43,346 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731751108989.7c7f87a6124bfe2f4049ee7803e0f98c., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c7f87a6124bfe2f4049ee7803e0f98c, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:58:43,346 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731751108989.7c7f87a6124bfe2f4049ee7803e0f98c., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 because the exception is null or not the one we care about 2024-11-16T09:58:43,346 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] ipc.CallRunner(138): callId: 83 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52420 deadline: 1731751183343, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c7f87a6124bfe2f4049ee7803e0f98c, server=037c58a19ddb,37267,1731751077844 2024-11-16T09:58:43,346 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731751108989.7c7f87a6124bfe2f4049ee7803e0f98c., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 , the old value is region=TestAcidGuarantees,,1731751108989.7c7f87a6124bfe2f4049ee7803e0f98c., hostname=037c58a19ddb,37267,1731751077844, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c7f87a6124bfe2f4049ee7803e0f98c, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:58:43,347 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731751108989.7c7f87a6124bfe2f4049ee7803e0f98c., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c7f87a6124bfe2f4049ee7803e0f98c, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:58:43,347 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731751108989.7c7f87a6124bfe2f4049ee7803e0f98c., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 because the exception is null or not the one we care about 2024-11-16T09:58:43,352 INFO [RS:0;037c58a19ddb:37267-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 7c7f87a6124bfe2f4049ee7803e0f98c#C#compaction#174 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-16T09:58:43,353 DEBUG [RS:0;037c58a19ddb:37267-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/7c7f87a6124bfe2f4049ee7803e0f98c/.tmp/C/4890337659474a46a61bf83aba890fb0 is 50, key is test_row_0/C:col10/1731751121633/Put/seqid=0 2024-11-16T09:58:43,365 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37591 is added to blk_1073742030_1206 (size=12629) 2024-11-16T09:58:43,375 DEBUG [RS:0;037c58a19ddb:37267-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/7c7f87a6124bfe2f4049ee7803e0f98c/.tmp/C/4890337659474a46a61bf83aba890fb0 as hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/7c7f87a6124bfe2f4049ee7803e0f98c/C/4890337659474a46a61bf83aba890fb0 2024-11-16T09:58:43,381 INFO [RS:0;037c58a19ddb:37267-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 7c7f87a6124bfe2f4049ee7803e0f98c/C of 7c7f87a6124bfe2f4049ee7803e0f98c into 4890337659474a46a61bf83aba890fb0(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-16T09:58:43,381 DEBUG [RS:0;037c58a19ddb:37267-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 7c7f87a6124bfe2f4049ee7803e0f98c: 2024-11-16T09:58:43,381 INFO [RS:0;037c58a19ddb:37267-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731751108989.7c7f87a6124bfe2f4049ee7803e0f98c., storeName=7c7f87a6124bfe2f4049ee7803e0f98c/C, priority=13, startTime=1731751122875; duration=0sec 2024-11-16T09:58:43,381 DEBUG [RS:0;037c58a19ddb:37267-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-16T09:58:43,381 DEBUG [RS:0;037c58a19ddb:37267-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 7c7f87a6124bfe2f4049ee7803e0f98c:C 2024-11-16T09:58:43,384 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=229, memsize=29.1 K, hasBloomFilter=true, into tmp file hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/7c7f87a6124bfe2f4049ee7803e0f98c/.tmp/A/2049d15304474f38af50e9ff363d240f 2024-11-16T09:58:43,394 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/7c7f87a6124bfe2f4049ee7803e0f98c/.tmp/B/a59127e94a214b4db7e66865fc9a2d7a is 50, key is test_row_0/B:col10/1731751122945/Put/seqid=0 2024-11-16T09:58:43,404 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37591 is added to blk_1073742031_1207 (size=12151) 2024-11-16T09:58:43,404 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=229 (bloomFilter=true), to=hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/7c7f87a6124bfe2f4049ee7803e0f98c/.tmp/B/a59127e94a214b4db7e66865fc9a2d7a 2024-11-16T09:58:43,414 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=37267 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=45 2024-11-16T09:58:43,414 DEBUG [RS_FLUSH_OPERATIONS-regionserver/037c58a19ddb:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731751108989.7c7f87a6124bfe2f4049ee7803e0f98c. 2024-11-16T09:58:43,414 DEBUG [RS_FLUSH_OPERATIONS-regionserver/037c58a19ddb:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731751108989.7c7f87a6124bfe2f4049ee7803e0f98c. as already flushing 2024-11-16T09:58:43,414 DEBUG [RS_FLUSH_OPERATIONS-regionserver/037c58a19ddb:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731751108989.7c7f87a6124bfe2f4049ee7803e0f98c. 2024-11-16T09:58:43,414 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/7c7f87a6124bfe2f4049ee7803e0f98c/.tmp/C/e1471e6f23124679a598ae6112b8b618 is 50, key is test_row_0/C:col10/1731751122945/Put/seqid=0 2024-11-16T09:58:43,414 ERROR [RS_FLUSH_OPERATIONS-regionserver/037c58a19ddb:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] handler.RSProcedureHandler(58): pid=45 java.io.IOException: Unable to complete flush {ENCODED => 7c7f87a6124bfe2f4049ee7803e0f98c, NAME => 'TestAcidGuarantees,,1731751108989.7c7f87a6124bfe2f4049ee7803e0f98c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T09:58:43,415 DEBUG [RS_FLUSH_OPERATIONS-regionserver/037c58a19ddb:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=45 java.io.IOException: Unable to complete flush {ENCODED => 7c7f87a6124bfe2f4049ee7803e0f98c, NAME => 'TestAcidGuarantees,,1731751108989.7c7f87a6124bfe2f4049ee7803e0f98c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T09:58:43,416 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37877 {}] master.HMaster(4177): Remote procedure failed, pid=45 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 7c7f87a6124bfe2f4049ee7803e0f98c, NAME => 'TestAcidGuarantees,,1731751108989.7c7f87a6124bfe2f4049ee7803e0f98c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 7c7f87a6124bfe2f4049ee7803e0f98c, NAME => 'TestAcidGuarantees,,1731751108989.7c7f87a6124bfe2f4049ee7803e0f98c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T09:58:43,430 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37591 is added to blk_1073742032_1208 (size=12151) 2024-11-16T09:58:43,433 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=229 (bloomFilter=true), to=hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/7c7f87a6124bfe2f4049ee7803e0f98c/.tmp/C/e1471e6f23124679a598ae6112b8b618 2024-11-16T09:58:43,440 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/7c7f87a6124bfe2f4049ee7803e0f98c/.tmp/A/2049d15304474f38af50e9ff363d240f as hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/7c7f87a6124bfe2f4049ee7803e0f98c/A/2049d15304474f38af50e9ff363d240f 2024-11-16T09:58:43,444 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/7c7f87a6124bfe2f4049ee7803e0f98c/A/2049d15304474f38af50e9ff363d240f, entries=200, sequenceid=229, filesize=38.8 K 2024-11-16T09:58:43,445 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/7c7f87a6124bfe2f4049ee7803e0f98c/.tmp/B/a59127e94a214b4db7e66865fc9a2d7a as hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/7c7f87a6124bfe2f4049ee7803e0f98c/B/a59127e94a214b4db7e66865fc9a2d7a 2024-11-16T09:58:43,452 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/7c7f87a6124bfe2f4049ee7803e0f98c/B/a59127e94a214b4db7e66865fc9a2d7a, entries=150, sequenceid=229, filesize=11.9 K 2024-11-16T09:58:43,454 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/7c7f87a6124bfe2f4049ee7803e0f98c/.tmp/C/e1471e6f23124679a598ae6112b8b618 as hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/7c7f87a6124bfe2f4049ee7803e0f98c/C/e1471e6f23124679a598ae6112b8b618 2024-11-16T09:58:43,462 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/7c7f87a6124bfe2f4049ee7803e0f98c/C/e1471e6f23124679a598ae6112b8b618, entries=150, sequenceid=229, filesize=11.9 K 2024-11-16T09:58:43,463 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~87.22 KB/89310, heapSize ~229.22 KB/234720, currentSize=120.76 KB/123660 for 7c7f87a6124bfe2f4049ee7803e0f98c in 518ms, sequenceid=229, compaction requested=false 2024-11-16T09:58:43,463 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 7c7f87a6124bfe2f4049ee7803e0f98c: 2024-11-16T09:58:43,568 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=37267 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=45 2024-11-16T09:58:43,568 DEBUG [RS_FLUSH_OPERATIONS-regionserver/037c58a19ddb:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731751108989.7c7f87a6124bfe2f4049ee7803e0f98c. 2024-11-16T09:58:43,569 INFO [RS_FLUSH_OPERATIONS-regionserver/037c58a19ddb:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegion(2902): Flushing 7c7f87a6124bfe2f4049ee7803e0f98c 3/3 column families, dataSize=120.76 KB heapSize=317.16 KB 2024-11-16T09:58:43,594 DEBUG [RS_FLUSH_OPERATIONS-regionserver/037c58a19ddb:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241116c8fbfa4416f84c84bf534fe38bdd1430_7c7f87a6124bfe2f4049ee7803e0f98c is 50, key is test_row_0/A:col10/1731751123016/Put/seqid=0 2024-11-16T09:58:43,606 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37591 is added to blk_1073742033_1209 (size=12304) 2024-11-16T09:58:43,646 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] regionserver.HRegion(8855): Flush requested on 7c7f87a6124bfe2f4049ee7803e0f98c 2024-11-16T09:58:43,646 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731751108989.7c7f87a6124bfe2f4049ee7803e0f98c. as already flushing 2024-11-16T09:58:43,694 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c7f87a6124bfe2f4049ee7803e0f98c, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T09:58:43,694 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] ipc.CallRunner(138): callId: 93 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52400 deadline: 1731751183689, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c7f87a6124bfe2f4049ee7803e0f98c, server=037c58a19ddb,37267,1731751077844 2024-11-16T09:58:43,694 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c7f87a6124bfe2f4049ee7803e0f98c, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T09:58:43,694 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731751108989.7c7f87a6124bfe2f4049ee7803e0f98c., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 , the old value is region=TestAcidGuarantees,,1731751108989.7c7f87a6124bfe2f4049ee7803e0f98c., hostname=037c58a19ddb,37267,1731751077844, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c7f87a6124bfe2f4049ee7803e0f98c, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:58:43,694 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731751108989.7c7f87a6124bfe2f4049ee7803e0f98c., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c7f87a6124bfe2f4049ee7803e0f98c, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:58:43,694 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] ipc.CallRunner(138): callId: 91 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52396 deadline: 1731751183689, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c7f87a6124bfe2f4049ee7803e0f98c, server=037c58a19ddb,37267,1731751077844 2024-11-16T09:58:43,694 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731751108989.7c7f87a6124bfe2f4049ee7803e0f98c., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 because the exception is null or not the one we care about 2024-11-16T09:58:43,695 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731751108989.7c7f87a6124bfe2f4049ee7803e0f98c., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 , the old value is region=TestAcidGuarantees,,1731751108989.7c7f87a6124bfe2f4049ee7803e0f98c., hostname=037c58a19ddb,37267,1731751077844, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c7f87a6124bfe2f4049ee7803e0f98c, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:58:43,695 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731751108989.7c7f87a6124bfe2f4049ee7803e0f98c., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c7f87a6124bfe2f4049ee7803e0f98c, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:58:43,695 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731751108989.7c7f87a6124bfe2f4049ee7803e0f98c., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 because the exception is null or not the one we care about 2024-11-16T09:58:43,700 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c7f87a6124bfe2f4049ee7803e0f98c, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T09:58:43,700 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] ipc.CallRunner(138): callId: 75 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52392 deadline: 1731751183693, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c7f87a6124bfe2f4049ee7803e0f98c, server=037c58a19ddb,37267,1731751077844 2024-11-16T09:58:43,700 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c7f87a6124bfe2f4049ee7803e0f98c, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T09:58:43,700 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731751108989.7c7f87a6124bfe2f4049ee7803e0f98c., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 , the old value is region=TestAcidGuarantees,,1731751108989.7c7f87a6124bfe2f4049ee7803e0f98c., hostname=037c58a19ddb,37267,1731751077844, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c7f87a6124bfe2f4049ee7803e0f98c, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:58:43,700 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] ipc.CallRunner(138): callId: 86 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52420 deadline: 1731751183694, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c7f87a6124bfe2f4049ee7803e0f98c, server=037c58a19ddb,37267,1731751077844 2024-11-16T09:58:43,700 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731751108989.7c7f87a6124bfe2f4049ee7803e0f98c., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c7f87a6124bfe2f4049ee7803e0f98c, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:58:43,700 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731751108989.7c7f87a6124bfe2f4049ee7803e0f98c., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 because the exception is null or not the one we care about 2024-11-16T09:58:43,700 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731751108989.7c7f87a6124bfe2f4049ee7803e0f98c., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 , the old value is region=TestAcidGuarantees,,1731751108989.7c7f87a6124bfe2f4049ee7803e0f98c., hostname=037c58a19ddb,37267,1731751077844, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c7f87a6124bfe2f4049ee7803e0f98c, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:58:43,701 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731751108989.7c7f87a6124bfe2f4049ee7803e0f98c., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c7f87a6124bfe2f4049ee7803e0f98c, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:58:43,701 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731751108989.7c7f87a6124bfe2f4049ee7803e0f98c., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 because the exception is null or not the one we care about 2024-11-16T09:58:43,744 DEBUG [RS:0;037c58a19ddb:37267-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/7c7f87a6124bfe2f4049ee7803e0f98c/.tmp/A/6b3a4c4820f0410ba008ad14ccf8af07 as hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/7c7f87a6124bfe2f4049ee7803e0f98c/A/6b3a4c4820f0410ba008ad14ccf8af07 2024-11-16T09:58:43,758 INFO [RS:0;037c58a19ddb:37267-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 7c7f87a6124bfe2f4049ee7803e0f98c/A of 7c7f87a6124bfe2f4049ee7803e0f98c into 6b3a4c4820f0410ba008ad14ccf8af07(size=30.8 K), total size for store is 69.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-16T09:58:43,758 DEBUG [RS:0;037c58a19ddb:37267-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 7c7f87a6124bfe2f4049ee7803e0f98c: 2024-11-16T09:58:43,758 INFO [RS:0;037c58a19ddb:37267-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731751108989.7c7f87a6124bfe2f4049ee7803e0f98c., storeName=7c7f87a6124bfe2f4049ee7803e0f98c/A, priority=13, startTime=1731751122874; duration=0sec 2024-11-16T09:58:43,758 DEBUG [RS:0;037c58a19ddb:37267-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-16T09:58:43,758 DEBUG [RS:0;037c58a19ddb:37267-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 7c7f87a6124bfe2f4049ee7803e0f98c:A 2024-11-16T09:58:43,783 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37877 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=44 2024-11-16T09:58:43,808 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c7f87a6124bfe2f4049ee7803e0f98c, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T09:58:43,808 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] ipc.CallRunner(138): callId: 94 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52400 deadline: 1731751183802, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c7f87a6124bfe2f4049ee7803e0f98c, server=037c58a19ddb,37267,1731751077844 2024-11-16T09:58:43,808 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c7f87a6124bfe2f4049ee7803e0f98c, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T09:58:43,808 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731751108989.7c7f87a6124bfe2f4049ee7803e0f98c., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 , the old value is region=TestAcidGuarantees,,1731751108989.7c7f87a6124bfe2f4049ee7803e0f98c., hostname=037c58a19ddb,37267,1731751077844, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c7f87a6124bfe2f4049ee7803e0f98c, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:58:43,808 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] ipc.CallRunner(138): callId: 76 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52392 deadline: 1731751183803, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c7f87a6124bfe2f4049ee7803e0f98c, server=037c58a19ddb,37267,1731751077844 2024-11-16T09:58:43,808 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731751108989.7c7f87a6124bfe2f4049ee7803e0f98c., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c7f87a6124bfe2f4049ee7803e0f98c, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:58:43,808 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731751108989.7c7f87a6124bfe2f4049ee7803e0f98c., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 because the exception is null or not the one we care about 2024-11-16T09:58:43,809 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731751108989.7c7f87a6124bfe2f4049ee7803e0f98c., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 , the old value is region=TestAcidGuarantees,,1731751108989.7c7f87a6124bfe2f4049ee7803e0f98c., hostname=037c58a19ddb,37267,1731751077844, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c7f87a6124bfe2f4049ee7803e0f98c, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:58:43,809 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c7f87a6124bfe2f4049ee7803e0f98c, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T09:58:43,809 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731751108989.7c7f87a6124bfe2f4049ee7803e0f98c., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c7f87a6124bfe2f4049ee7803e0f98c, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:58:43,809 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731751108989.7c7f87a6124bfe2f4049ee7803e0f98c., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 because the exception is null or not the one we care about 2024-11-16T09:58:43,809 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] ipc.CallRunner(138): callId: 92 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52396 deadline: 1731751183803, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c7f87a6124bfe2f4049ee7803e0f98c, server=037c58a19ddb,37267,1731751077844 2024-11-16T09:58:43,809 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c7f87a6124bfe2f4049ee7803e0f98c, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T09:58:43,809 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731751108989.7c7f87a6124bfe2f4049ee7803e0f98c., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 , the old value is region=TestAcidGuarantees,,1731751108989.7c7f87a6124bfe2f4049ee7803e0f98c., hostname=037c58a19ddb,37267,1731751077844, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c7f87a6124bfe2f4049ee7803e0f98c, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:58:43,809 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] ipc.CallRunner(138): callId: 87 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52420 deadline: 1731751183803, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c7f87a6124bfe2f4049ee7803e0f98c, server=037c58a19ddb,37267,1731751077844 2024-11-16T09:58:43,809 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731751108989.7c7f87a6124bfe2f4049ee7803e0f98c., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c7f87a6124bfe2f4049ee7803e0f98c, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:58:43,809 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731751108989.7c7f87a6124bfe2f4049ee7803e0f98c., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 because the exception is null or not the one we care about 2024-11-16T09:58:43,810 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731751108989.7c7f87a6124bfe2f4049ee7803e0f98c., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 , the old value is region=TestAcidGuarantees,,1731751108989.7c7f87a6124bfe2f4049ee7803e0f98c., hostname=037c58a19ddb,37267,1731751077844, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c7f87a6124bfe2f4049ee7803e0f98c, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:58:43,810 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731751108989.7c7f87a6124bfe2f4049ee7803e0f98c., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c7f87a6124bfe2f4049ee7803e0f98c, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:58:43,810 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731751108989.7c7f87a6124bfe2f4049ee7803e0f98c., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 because the exception is null or not the one we care about 2024-11-16T09:58:44,006 DEBUG [RS_FLUSH_OPERATIONS-regionserver/037c58a19ddb:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T09:58:44,013 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c7f87a6124bfe2f4049ee7803e0f98c, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T09:58:44,014 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] ipc.CallRunner(138): callId: 88 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52420 deadline: 1731751184013, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c7f87a6124bfe2f4049ee7803e0f98c, server=037c58a19ddb,37267,1731751077844 2024-11-16T09:58:44,014 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c7f87a6124bfe2f4049ee7803e0f98c, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T09:58:44,014 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] ipc.CallRunner(138): callId: 93 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52396 deadline: 1731751184013, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c7f87a6124bfe2f4049ee7803e0f98c, server=037c58a19ddb,37267,1731751077844 2024-11-16T09:58:44,014 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731751108989.7c7f87a6124bfe2f4049ee7803e0f98c., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 , the old value is region=TestAcidGuarantees,,1731751108989.7c7f87a6124bfe2f4049ee7803e0f98c., hostname=037c58a19ddb,37267,1731751077844, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c7f87a6124bfe2f4049ee7803e0f98c, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:58:44,014 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731751108989.7c7f87a6124bfe2f4049ee7803e0f98c., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c7f87a6124bfe2f4049ee7803e0f98c, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:58:44,014 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731751108989.7c7f87a6124bfe2f4049ee7803e0f98c., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 because the exception is null or not the one we care about 2024-11-16T09:58:44,014 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c7f87a6124bfe2f4049ee7803e0f98c, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T09:58:44,014 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731751108989.7c7f87a6124bfe2f4049ee7803e0f98c., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 , the old value is region=TestAcidGuarantees,,1731751108989.7c7f87a6124bfe2f4049ee7803e0f98c., hostname=037c58a19ddb,37267,1731751077844, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c7f87a6124bfe2f4049ee7803e0f98c, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:58:44,014 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731751108989.7c7f87a6124bfe2f4049ee7803e0f98c., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c7f87a6124bfe2f4049ee7803e0f98c, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:58:44,015 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] ipc.CallRunner(138): callId: 95 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52400 deadline: 1731751184014, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c7f87a6124bfe2f4049ee7803e0f98c, server=037c58a19ddb,37267,1731751077844 2024-11-16T09:58:44,015 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731751108989.7c7f87a6124bfe2f4049ee7803e0f98c., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 because the exception is null or not the one we care about 2024-11-16T09:58:44,015 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c7f87a6124bfe2f4049ee7803e0f98c, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T09:58:44,015 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] ipc.CallRunner(138): callId: 77 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52392 deadline: 1731751184015, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c7f87a6124bfe2f4049ee7803e0f98c, server=037c58a19ddb,37267,1731751077844 2024-11-16T09:58:44,015 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731751108989.7c7f87a6124bfe2f4049ee7803e0f98c., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 , the old value is region=TestAcidGuarantees,,1731751108989.7c7f87a6124bfe2f4049ee7803e0f98c., hostname=037c58a19ddb,37267,1731751077844, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c7f87a6124bfe2f4049ee7803e0f98c, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:58:44,015 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731751108989.7c7f87a6124bfe2f4049ee7803e0f98c., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c7f87a6124bfe2f4049ee7803e0f98c, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:58:44,015 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731751108989.7c7f87a6124bfe2f4049ee7803e0f98c., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 because the exception is null or not the one we care about 2024-11-16T09:58:44,016 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731751108989.7c7f87a6124bfe2f4049ee7803e0f98c., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 , the old value is region=TestAcidGuarantees,,1731751108989.7c7f87a6124bfe2f4049ee7803e0f98c., hostname=037c58a19ddb,37267,1731751077844, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c7f87a6124bfe2f4049ee7803e0f98c, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:58:44,016 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731751108989.7c7f87a6124bfe2f4049ee7803e0f98c., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c7f87a6124bfe2f4049ee7803e0f98c, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:58:44,016 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731751108989.7c7f87a6124bfe2f4049ee7803e0f98c., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 because the exception is null or not the one we care about 2024-11-16T09:58:44,020 INFO [RS_FLUSH_OPERATIONS-regionserver/037c58a19ddb:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241116c8fbfa4416f84c84bf534fe38bdd1430_7c7f87a6124bfe2f4049ee7803e0f98c to hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241116c8fbfa4416f84c84bf534fe38bdd1430_7c7f87a6124bfe2f4049ee7803e0f98c 2024-11-16T09:58:44,021 DEBUG [RS_FLUSH_OPERATIONS-regionserver/037c58a19ddb:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/7c7f87a6124bfe2f4049ee7803e0f98c/.tmp/A/15c9a113751c448ab4ea0ce9821776a1, store: [table=TestAcidGuarantees family=A region=7c7f87a6124bfe2f4049ee7803e0f98c] 2024-11-16T09:58:44,021 DEBUG [RS_FLUSH_OPERATIONS-regionserver/037c58a19ddb:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/7c7f87a6124bfe2f4049ee7803e0f98c/.tmp/A/15c9a113751c448ab4ea0ce9821776a1 is 175, key is test_row_0/A:col10/1731751123016/Put/seqid=0 2024-11-16T09:58:44,042 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37591 is added to blk_1073742034_1210 (size=31105) 2024-11-16T09:58:44,042 INFO [RS_FLUSH_OPERATIONS-regionserver/037c58a19ddb:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=252, memsize=40.3 K, hasBloomFilter=true, into tmp file hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/7c7f87a6124bfe2f4049ee7803e0f98c/.tmp/A/15c9a113751c448ab4ea0ce9821776a1 2024-11-16T09:58:44,062 DEBUG [RS_FLUSH_OPERATIONS-regionserver/037c58a19ddb:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/7c7f87a6124bfe2f4049ee7803e0f98c/.tmp/B/a45924af8cf74348bbdc6ab01952e6e7 is 50, key is test_row_0/B:col10/1731751123016/Put/seqid=0 2024-11-16T09:58:44,078 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37591 is added to blk_1073742035_1211 (size=12151) 2024-11-16T09:58:44,079 INFO [RS_FLUSH_OPERATIONS-regionserver/037c58a19ddb:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=252 (bloomFilter=true), to=hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/7c7f87a6124bfe2f4049ee7803e0f98c/.tmp/B/a45924af8cf74348bbdc6ab01952e6e7 2024-11-16T09:58:44,091 DEBUG [RS_FLUSH_OPERATIONS-regionserver/037c58a19ddb:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/7c7f87a6124bfe2f4049ee7803e0f98c/.tmp/C/3064c932d4fc4dc49cae309d5c3bdfbe is 50, key is test_row_0/C:col10/1731751123016/Put/seqid=0 2024-11-16T09:58:44,101 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37591 is added to blk_1073742036_1212 (size=12151) 2024-11-16T09:58:44,107 INFO [RS_FLUSH_OPERATIONS-regionserver/037c58a19ddb:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=252 (bloomFilter=true), to=hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/7c7f87a6124bfe2f4049ee7803e0f98c/.tmp/C/3064c932d4fc4dc49cae309d5c3bdfbe 2024-11-16T09:58:44,113 DEBUG [RS_FLUSH_OPERATIONS-regionserver/037c58a19ddb:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/7c7f87a6124bfe2f4049ee7803e0f98c/.tmp/A/15c9a113751c448ab4ea0ce9821776a1 as hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/7c7f87a6124bfe2f4049ee7803e0f98c/A/15c9a113751c448ab4ea0ce9821776a1 2024-11-16T09:58:44,119 INFO [RS_FLUSH_OPERATIONS-regionserver/037c58a19ddb:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/7c7f87a6124bfe2f4049ee7803e0f98c/A/15c9a113751c448ab4ea0ce9821776a1, entries=150, sequenceid=252, filesize=30.4 K 2024-11-16T09:58:44,129 DEBUG [RS_FLUSH_OPERATIONS-regionserver/037c58a19ddb:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/7c7f87a6124bfe2f4049ee7803e0f98c/.tmp/B/a45924af8cf74348bbdc6ab01952e6e7 as hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/7c7f87a6124bfe2f4049ee7803e0f98c/B/a45924af8cf74348bbdc6ab01952e6e7 2024-11-16T09:58:44,135 INFO [RS_FLUSH_OPERATIONS-regionserver/037c58a19ddb:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/7c7f87a6124bfe2f4049ee7803e0f98c/B/a45924af8cf74348bbdc6ab01952e6e7, entries=150, sequenceid=252, filesize=11.9 K 2024-11-16T09:58:44,137 DEBUG [RS_FLUSH_OPERATIONS-regionserver/037c58a19ddb:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/7c7f87a6124bfe2f4049ee7803e0f98c/.tmp/C/3064c932d4fc4dc49cae309d5c3bdfbe as hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/7c7f87a6124bfe2f4049ee7803e0f98c/C/3064c932d4fc4dc49cae309d5c3bdfbe 2024-11-16T09:58:44,145 INFO [RS_FLUSH_OPERATIONS-regionserver/037c58a19ddb:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/7c7f87a6124bfe2f4049ee7803e0f98c/C/3064c932d4fc4dc49cae309d5c3bdfbe, entries=150, sequenceid=252, filesize=11.9 K 2024-11-16T09:58:44,147 INFO [RS_FLUSH_OPERATIONS-regionserver/037c58a19ddb:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegion(3140): Finished flush of dataSize ~120.76 KB/123660, heapSize ~317.11 KB/324720, currentSize=80.51 KB/82440 for 7c7f87a6124bfe2f4049ee7803e0f98c in 579ms, sequenceid=252, compaction requested=true 2024-11-16T09:58:44,147 DEBUG [RS_FLUSH_OPERATIONS-regionserver/037c58a19ddb:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegion(2603): Flush status journal for 7c7f87a6124bfe2f4049ee7803e0f98c: 2024-11-16T09:58:44,147 DEBUG [RS_FLUSH_OPERATIONS-regionserver/037c58a19ddb:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731751108989.7c7f87a6124bfe2f4049ee7803e0f98c. 2024-11-16T09:58:44,147 DEBUG [RS_FLUSH_OPERATIONS-regionserver/037c58a19ddb:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=45 2024-11-16T09:58:44,147 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37877 {}] master.HMaster(4169): Remote procedure done, pid=45 2024-11-16T09:58:44,152 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=45, resume processing ppid=44 2024-11-16T09:58:44,152 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=45, ppid=44, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.5020 sec 2024-11-16T09:58:44,156 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=44, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=44, table=TestAcidGuarantees in 1.5100 sec 2024-11-16T09:58:44,326 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] regionserver.HRegion(8855): Flush requested on 7c7f87a6124bfe2f4049ee7803e0f98c 2024-11-16T09:58:44,326 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-16T09:58:44,326 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 7c7f87a6124bfe2f4049ee7803e0f98c 3/3 column families, dataSize=93.93 KB heapSize=246.84 KB 2024-11-16T09:58:44,349 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111624023427d774423487434234fccac8f7_7c7f87a6124bfe2f4049ee7803e0f98c is 50, key is test_row_0/A:col10/1731751124326/Put/seqid=0 2024-11-16T09:58:44,364 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c7f87a6124bfe2f4049ee7803e0f98c, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T09:58:44,364 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] ipc.CallRunner(138): callId: 98 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52396 deadline: 1731751184354, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c7f87a6124bfe2f4049ee7803e0f98c, server=037c58a19ddb,37267,1731751077844 2024-11-16T09:58:44,365 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731751108989.7c7f87a6124bfe2f4049ee7803e0f98c., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 , the old value is region=TestAcidGuarantees,,1731751108989.7c7f87a6124bfe2f4049ee7803e0f98c., hostname=037c58a19ddb,37267,1731751077844, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c7f87a6124bfe2f4049ee7803e0f98c, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:58:44,365 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731751108989.7c7f87a6124bfe2f4049ee7803e0f98c., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c7f87a6124bfe2f4049ee7803e0f98c, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:58:44,365 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731751108989.7c7f87a6124bfe2f4049ee7803e0f98c., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 because the exception is null or not the one we care about 2024-11-16T09:58:44,365 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c7f87a6124bfe2f4049ee7803e0f98c, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T09:58:44,366 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] ipc.CallRunner(138): callId: 93 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52420 deadline: 1731751184356, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c7f87a6124bfe2f4049ee7803e0f98c, server=037c58a19ddb,37267,1731751077844 2024-11-16T09:58:44,366 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731751108989.7c7f87a6124bfe2f4049ee7803e0f98c., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 , the old value is region=TestAcidGuarantees,,1731751108989.7c7f87a6124bfe2f4049ee7803e0f98c., hostname=037c58a19ddb,37267,1731751077844, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c7f87a6124bfe2f4049ee7803e0f98c, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:58:44,366 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731751108989.7c7f87a6124bfe2f4049ee7803e0f98c., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c7f87a6124bfe2f4049ee7803e0f98c, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:58:44,366 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731751108989.7c7f87a6124bfe2f4049ee7803e0f98c., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 because the exception is null or not the one we care about 2024-11-16T09:58:44,366 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c7f87a6124bfe2f4049ee7803e0f98c, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T09:58:44,366 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] ipc.CallRunner(138): callId: 83 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52392 deadline: 1731751184358, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c7f87a6124bfe2f4049ee7803e0f98c, server=037c58a19ddb,37267,1731751077844 2024-11-16T09:58:44,366 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731751108989.7c7f87a6124bfe2f4049ee7803e0f98c., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 , the old value is region=TestAcidGuarantees,,1731751108989.7c7f87a6124bfe2f4049ee7803e0f98c., hostname=037c58a19ddb,37267,1731751077844, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c7f87a6124bfe2f4049ee7803e0f98c, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:58:44,366 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731751108989.7c7f87a6124bfe2f4049ee7803e0f98c., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c7f87a6124bfe2f4049ee7803e0f98c, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:58:44,367 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731751108989.7c7f87a6124bfe2f4049ee7803e0f98c., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 because the exception is null or not the one we care about 2024-11-16T09:58:44,367 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c7f87a6124bfe2f4049ee7803e0f98c, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T09:58:44,367 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] ipc.CallRunner(138): callId: 101 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52400 deadline: 1731751184361, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c7f87a6124bfe2f4049ee7803e0f98c, server=037c58a19ddb,37267,1731751077844 2024-11-16T09:58:44,367 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731751108989.7c7f87a6124bfe2f4049ee7803e0f98c., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 , the old value is region=TestAcidGuarantees,,1731751108989.7c7f87a6124bfe2f4049ee7803e0f98c., hostname=037c58a19ddb,37267,1731751077844, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c7f87a6124bfe2f4049ee7803e0f98c, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:58:44,367 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731751108989.7c7f87a6124bfe2f4049ee7803e0f98c., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c7f87a6124bfe2f4049ee7803e0f98c, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:58:44,367 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731751108989.7c7f87a6124bfe2f4049ee7803e0f98c., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 because the exception is null or not the one we care about 2024-11-16T09:58:44,398 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37591 is added to blk_1073742037_1213 (size=14994) 2024-11-16T09:58:44,476 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c7f87a6124bfe2f4049ee7803e0f98c, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T09:58:44,477 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c7f87a6124bfe2f4049ee7803e0f98c, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T09:58:44,477 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] ipc.CallRunner(138): callId: 99 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52396 deadline: 1731751184473, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c7f87a6124bfe2f4049ee7803e0f98c, server=037c58a19ddb,37267,1731751077844 2024-11-16T09:58:44,477 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] ipc.CallRunner(138): callId: 94 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52420 deadline: 1731751184473, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c7f87a6124bfe2f4049ee7803e0f98c, server=037c58a19ddb,37267,1731751077844 2024-11-16T09:58:44,477 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731751108989.7c7f87a6124bfe2f4049ee7803e0f98c., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 , the old value is region=TestAcidGuarantees,,1731751108989.7c7f87a6124bfe2f4049ee7803e0f98c., hostname=037c58a19ddb,37267,1731751077844, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c7f87a6124bfe2f4049ee7803e0f98c, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:58:44,477 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731751108989.7c7f87a6124bfe2f4049ee7803e0f98c., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c7f87a6124bfe2f4049ee7803e0f98c, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:58:44,477 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c7f87a6124bfe2f4049ee7803e0f98c, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T09:58:44,477 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731751108989.7c7f87a6124bfe2f4049ee7803e0f98c., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 because the exception is null or not the one we care about 2024-11-16T09:58:44,477 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c7f87a6124bfe2f4049ee7803e0f98c, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T09:58:44,477 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] ipc.CallRunner(138): callId: 102 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52400 deadline: 1731751184473, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c7f87a6124bfe2f4049ee7803e0f98c, server=037c58a19ddb,37267,1731751077844 2024-11-16T09:58:44,477 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] ipc.CallRunner(138): callId: 84 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52392 deadline: 1731751184473, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c7f87a6124bfe2f4049ee7803e0f98c, server=037c58a19ddb,37267,1731751077844 2024-11-16T09:58:44,478 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731751108989.7c7f87a6124bfe2f4049ee7803e0f98c., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 , the old value is region=TestAcidGuarantees,,1731751108989.7c7f87a6124bfe2f4049ee7803e0f98c., hostname=037c58a19ddb,37267,1731751077844, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c7f87a6124bfe2f4049ee7803e0f98c, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:58:44,478 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731751108989.7c7f87a6124bfe2f4049ee7803e0f98c., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c7f87a6124bfe2f4049ee7803e0f98c, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:58:44,478 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731751108989.7c7f87a6124bfe2f4049ee7803e0f98c., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 because the exception is null or not the one we care about 2024-11-16T09:58:44,478 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731751108989.7c7f87a6124bfe2f4049ee7803e0f98c., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 , the old value is region=TestAcidGuarantees,,1731751108989.7c7f87a6124bfe2f4049ee7803e0f98c., hostname=037c58a19ddb,37267,1731751077844, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c7f87a6124bfe2f4049ee7803e0f98c, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:58:44,478 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731751108989.7c7f87a6124bfe2f4049ee7803e0f98c., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c7f87a6124bfe2f4049ee7803e0f98c, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:58:44,478 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731751108989.7c7f87a6124bfe2f4049ee7803e0f98c., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 because the exception is null or not the one we care about 2024-11-16T09:58:44,478 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731751108989.7c7f87a6124bfe2f4049ee7803e0f98c., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 , the old value is region=TestAcidGuarantees,,1731751108989.7c7f87a6124bfe2f4049ee7803e0f98c., hostname=037c58a19ddb,37267,1731751077844, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c7f87a6124bfe2f4049ee7803e0f98c, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:58:44,478 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731751108989.7c7f87a6124bfe2f4049ee7803e0f98c., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c7f87a6124bfe2f4049ee7803e0f98c, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:58:44,478 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731751108989.7c7f87a6124bfe2f4049ee7803e0f98c., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 because the exception is null or not the one we care about 2024-11-16T09:58:44,687 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c7f87a6124bfe2f4049ee7803e0f98c, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T09:58:44,688 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] ipc.CallRunner(138): callId: 85 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52392 deadline: 1731751184683, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c7f87a6124bfe2f4049ee7803e0f98c, server=037c58a19ddb,37267,1731751077844 2024-11-16T09:58:44,688 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731751108989.7c7f87a6124bfe2f4049ee7803e0f98c., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 , the old value is region=TestAcidGuarantees,,1731751108989.7c7f87a6124bfe2f4049ee7803e0f98c., hostname=037c58a19ddb,37267,1731751077844, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c7f87a6124bfe2f4049ee7803e0f98c, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:58:44,688 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731751108989.7c7f87a6124bfe2f4049ee7803e0f98c., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c7f87a6124bfe2f4049ee7803e0f98c, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:58:44,688 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731751108989.7c7f87a6124bfe2f4049ee7803e0f98c., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 because the exception is null or not the one we care about 2024-11-16T09:58:44,691 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c7f87a6124bfe2f4049ee7803e0f98c, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T09:58:44,692 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] ipc.CallRunner(138): callId: 103 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52400 deadline: 1731751184684, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c7f87a6124bfe2f4049ee7803e0f98c, server=037c58a19ddb,37267,1731751077844 2024-11-16T09:58:44,692 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c7f87a6124bfe2f4049ee7803e0f98c, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T09:58:44,692 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] ipc.CallRunner(138): callId: 95 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52420 deadline: 1731751184684, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c7f87a6124bfe2f4049ee7803e0f98c, server=037c58a19ddb,37267,1731751077844 2024-11-16T09:58:44,692 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c7f87a6124bfe2f4049ee7803e0f98c, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T09:58:44,692 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731751108989.7c7f87a6124bfe2f4049ee7803e0f98c., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 , the old value is region=TestAcidGuarantees,,1731751108989.7c7f87a6124bfe2f4049ee7803e0f98c., hostname=037c58a19ddb,37267,1731751077844, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c7f87a6124bfe2f4049ee7803e0f98c, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:58:44,692 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] ipc.CallRunner(138): callId: 100 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52396 deadline: 1731751184684, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c7f87a6124bfe2f4049ee7803e0f98c, server=037c58a19ddb,37267,1731751077844 2024-11-16T09:58:44,692 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731751108989.7c7f87a6124bfe2f4049ee7803e0f98c., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c7f87a6124bfe2f4049ee7803e0f98c, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:58:44,692 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731751108989.7c7f87a6124bfe2f4049ee7803e0f98c., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 because the exception is null or not the one we care about 2024-11-16T09:58:44,692 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731751108989.7c7f87a6124bfe2f4049ee7803e0f98c., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 , the old value is region=TestAcidGuarantees,,1731751108989.7c7f87a6124bfe2f4049ee7803e0f98c., hostname=037c58a19ddb,37267,1731751077844, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c7f87a6124bfe2f4049ee7803e0f98c, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:58:44,692 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731751108989.7c7f87a6124bfe2f4049ee7803e0f98c., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c7f87a6124bfe2f4049ee7803e0f98c, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:58:44,692 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731751108989.7c7f87a6124bfe2f4049ee7803e0f98c., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 , the old value is region=TestAcidGuarantees,,1731751108989.7c7f87a6124bfe2f4049ee7803e0f98c., hostname=037c58a19ddb,37267,1731751077844, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c7f87a6124bfe2f4049ee7803e0f98c, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:58:44,693 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731751108989.7c7f87a6124bfe2f4049ee7803e0f98c., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 because the exception is null or not the one we care about 2024-11-16T09:58:44,693 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731751108989.7c7f87a6124bfe2f4049ee7803e0f98c., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c7f87a6124bfe2f4049ee7803e0f98c, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:58:44,693 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731751108989.7c7f87a6124bfe2f4049ee7803e0f98c., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 because the exception is null or not the one we care about 2024-11-16T09:58:44,793 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37877 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=44 2024-11-16T09:58:44,793 INFO [RPCClient-NioEventLoopGroup-4-7 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-16T09:58:44,793 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37877 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-16T09:58:44,794 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37877 {}] procedure2.ProcedureExecutor(1139): Stored pid=46, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=46, table=TestAcidGuarantees 2024-11-16T09:58:44,795 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37877 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=46 2024-11-16T09:58:44,795 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=46, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=46, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-16T09:58:44,796 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=46, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=46, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-16T09:58:44,796 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=47, ppid=46, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-16T09:58:44,799 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T09:58:44,804 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111624023427d774423487434234fccac8f7_7c7f87a6124bfe2f4049ee7803e0f98c to hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111624023427d774423487434234fccac8f7_7c7f87a6124bfe2f4049ee7803e0f98c 2024-11-16T09:58:44,804 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/7c7f87a6124bfe2f4049ee7803e0f98c/.tmp/A/36f2297c7adc4ecaaf1ca484ad85f45f, store: [table=TestAcidGuarantees family=A region=7c7f87a6124bfe2f4049ee7803e0f98c] 2024-11-16T09:58:44,805 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/7c7f87a6124bfe2f4049ee7803e0f98c/.tmp/A/36f2297c7adc4ecaaf1ca484ad85f45f is 175, key is test_row_0/A:col10/1731751124326/Put/seqid=0 2024-11-16T09:58:44,812 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37591 is added to blk_1073742038_1214 (size=39949) 2024-11-16T09:58:44,902 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37877 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=46 2024-11-16T09:58:44,948 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=37267 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=47 2024-11-16T09:58:44,948 DEBUG [RS_FLUSH_OPERATIONS-regionserver/037c58a19ddb:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731751108989.7c7f87a6124bfe2f4049ee7803e0f98c. 2024-11-16T09:58:44,949 DEBUG [RS_FLUSH_OPERATIONS-regionserver/037c58a19ddb:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731751108989.7c7f87a6124bfe2f4049ee7803e0f98c. as already flushing 2024-11-16T09:58:44,949 DEBUG [RS_FLUSH_OPERATIONS-regionserver/037c58a19ddb:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731751108989.7c7f87a6124bfe2f4049ee7803e0f98c. 2024-11-16T09:58:44,949 ERROR [RS_FLUSH_OPERATIONS-regionserver/037c58a19ddb:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] handler.RSProcedureHandler(58): pid=47 java.io.IOException: Unable to complete flush {ENCODED => 7c7f87a6124bfe2f4049ee7803e0f98c, NAME => 'TestAcidGuarantees,,1731751108989.7c7f87a6124bfe2f4049ee7803e0f98c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T09:58:44,949 DEBUG [RS_FLUSH_OPERATIONS-regionserver/037c58a19ddb:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=47 java.io.IOException: Unable to complete flush {ENCODED => 7c7f87a6124bfe2f4049ee7803e0f98c, NAME => 'TestAcidGuarantees,,1731751108989.7c7f87a6124bfe2f4049ee7803e0f98c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T09:58:44,950 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37877 {}] master.HMaster(4177): Remote procedure failed, pid=47 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 7c7f87a6124bfe2f4049ee7803e0f98c, NAME => 'TestAcidGuarantees,,1731751108989.7c7f87a6124bfe2f4049ee7803e0f98c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 7c7f87a6124bfe2f4049ee7803e0f98c, NAME => 'TestAcidGuarantees,,1731751108989.7c7f87a6124bfe2f4049ee7803e0f98c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T09:58:44,994 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c7f87a6124bfe2f4049ee7803e0f98c, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T09:58:44,994 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] ipc.CallRunner(138): callId: 86 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52392 deadline: 1731751184993, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c7f87a6124bfe2f4049ee7803e0f98c, server=037c58a19ddb,37267,1731751077844 2024-11-16T09:58:44,994 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731751108989.7c7f87a6124bfe2f4049ee7803e0f98c., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 , the old value is region=TestAcidGuarantees,,1731751108989.7c7f87a6124bfe2f4049ee7803e0f98c., hostname=037c58a19ddb,37267,1731751077844, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c7f87a6124bfe2f4049ee7803e0f98c, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:58:44,995 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731751108989.7c7f87a6124bfe2f4049ee7803e0f98c., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c7f87a6124bfe2f4049ee7803e0f98c, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:58:44,995 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731751108989.7c7f87a6124bfe2f4049ee7803e0f98c., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 because the exception is null or not the one we care about 2024-11-16T09:58:45,004 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c7f87a6124bfe2f4049ee7803e0f98c, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T09:58:45,004 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] ipc.CallRunner(138): callId: 104 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52400 deadline: 1731751185002, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c7f87a6124bfe2f4049ee7803e0f98c, server=037c58a19ddb,37267,1731751077844 2024-11-16T09:58:45,004 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c7f87a6124bfe2f4049ee7803e0f98c, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T09:58:45,004 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731751108989.7c7f87a6124bfe2f4049ee7803e0f98c., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 , the old value is region=TestAcidGuarantees,,1731751108989.7c7f87a6124bfe2f4049ee7803e0f98c., hostname=037c58a19ddb,37267,1731751077844, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c7f87a6124bfe2f4049ee7803e0f98c, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:58:45,004 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731751108989.7c7f87a6124bfe2f4049ee7803e0f98c., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c7f87a6124bfe2f4049ee7803e0f98c, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:58:45,004 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] ipc.CallRunner(138): callId: 96 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52420 deadline: 1731751185003, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c7f87a6124bfe2f4049ee7803e0f98c, server=037c58a19ddb,37267,1731751077844 2024-11-16T09:58:45,004 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731751108989.7c7f87a6124bfe2f4049ee7803e0f98c., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 because the exception is null or not the one we care about 2024-11-16T09:58:45,005 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731751108989.7c7f87a6124bfe2f4049ee7803e0f98c., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 , the old value is region=TestAcidGuarantees,,1731751108989.7c7f87a6124bfe2f4049ee7803e0f98c., hostname=037c58a19ddb,37267,1731751077844, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c7f87a6124bfe2f4049ee7803e0f98c, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:58:45,005 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731751108989.7c7f87a6124bfe2f4049ee7803e0f98c., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c7f87a6124bfe2f4049ee7803e0f98c, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:58:45,005 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731751108989.7c7f87a6124bfe2f4049ee7803e0f98c., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 because the exception is null or not the one we care about 2024-11-16T09:58:45,005 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c7f87a6124bfe2f4049ee7803e0f98c, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T09:58:45,005 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] ipc.CallRunner(138): callId: 101 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52396 deadline: 1731751185003, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c7f87a6124bfe2f4049ee7803e0f98c, server=037c58a19ddb,37267,1731751077844 2024-11-16T09:58:45,005 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731751108989.7c7f87a6124bfe2f4049ee7803e0f98c., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 , the old value is region=TestAcidGuarantees,,1731751108989.7c7f87a6124bfe2f4049ee7803e0f98c., hostname=037c58a19ddb,37267,1731751077844, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c7f87a6124bfe2f4049ee7803e0f98c, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:58:45,005 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731751108989.7c7f87a6124bfe2f4049ee7803e0f98c., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c7f87a6124bfe2f4049ee7803e0f98c, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:58:45,005 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731751108989.7c7f87a6124bfe2f4049ee7803e0f98c., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 because the exception is null or not the one we care about 2024-11-16T09:58:45,102 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=37267 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=47 2024-11-16T09:58:45,102 DEBUG [RS_FLUSH_OPERATIONS-regionserver/037c58a19ddb:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731751108989.7c7f87a6124bfe2f4049ee7803e0f98c. 2024-11-16T09:58:45,102 DEBUG [RS_FLUSH_OPERATIONS-regionserver/037c58a19ddb:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731751108989.7c7f87a6124bfe2f4049ee7803e0f98c. as already flushing 2024-11-16T09:58:45,102 DEBUG [RS_FLUSH_OPERATIONS-regionserver/037c58a19ddb:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731751108989.7c7f87a6124bfe2f4049ee7803e0f98c. 2024-11-16T09:58:45,102 ERROR [RS_FLUSH_OPERATIONS-regionserver/037c58a19ddb:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] handler.RSProcedureHandler(58): pid=47 java.io.IOException: Unable to complete flush {ENCODED => 7c7f87a6124bfe2f4049ee7803e0f98c, NAME => 'TestAcidGuarantees,,1731751108989.7c7f87a6124bfe2f4049ee7803e0f98c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T09:58:45,102 DEBUG [RS_FLUSH_OPERATIONS-regionserver/037c58a19ddb:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=47 java.io.IOException: Unable to complete flush {ENCODED => 7c7f87a6124bfe2f4049ee7803e0f98c, NAME => 'TestAcidGuarantees,,1731751108989.7c7f87a6124bfe2f4049ee7803e0f98c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T09:58:45,103 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37877 {}] master.HMaster(4177): Remote procedure failed, pid=47 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 7c7f87a6124bfe2f4049ee7803e0f98c, NAME => 'TestAcidGuarantees,,1731751108989.7c7f87a6124bfe2f4049ee7803e0f98c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 7c7f87a6124bfe2f4049ee7803e0f98c, NAME => 'TestAcidGuarantees,,1731751108989.7c7f87a6124bfe2f4049ee7803e0f98c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T09:58:45,113 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37877 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=46 2024-11-16T09:58:45,213 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=271, memsize=33.5 K, hasBloomFilter=true, into tmp file hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/7c7f87a6124bfe2f4049ee7803e0f98c/.tmp/A/36f2297c7adc4ecaaf1ca484ad85f45f 2024-11-16T09:58:45,224 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/7c7f87a6124bfe2f4049ee7803e0f98c/.tmp/B/cdd6bdee112a48518295ab24f2e5e365 is 50, key is test_row_0/B:col10/1731751124326/Put/seqid=0 2024-11-16T09:58:45,228 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37591 is added to blk_1073742039_1215 (size=12301) 2024-11-16T09:58:45,230 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=271 (bloomFilter=true), to=hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/7c7f87a6124bfe2f4049ee7803e0f98c/.tmp/B/cdd6bdee112a48518295ab24f2e5e365 2024-11-16T09:58:45,255 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=37267 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=47 2024-11-16T09:58:45,255 DEBUG [RS_FLUSH_OPERATIONS-regionserver/037c58a19ddb:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731751108989.7c7f87a6124bfe2f4049ee7803e0f98c. 2024-11-16T09:58:45,255 DEBUG [RS_FLUSH_OPERATIONS-regionserver/037c58a19ddb:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731751108989.7c7f87a6124bfe2f4049ee7803e0f98c. as already flushing 2024-11-16T09:58:45,255 DEBUG [RS_FLUSH_OPERATIONS-regionserver/037c58a19ddb:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731751108989.7c7f87a6124bfe2f4049ee7803e0f98c. 2024-11-16T09:58:45,255 ERROR [RS_FLUSH_OPERATIONS-regionserver/037c58a19ddb:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] handler.RSProcedureHandler(58): pid=47 java.io.IOException: Unable to complete flush {ENCODED => 7c7f87a6124bfe2f4049ee7803e0f98c, NAME => 'TestAcidGuarantees,,1731751108989.7c7f87a6124bfe2f4049ee7803e0f98c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T09:58:45,256 DEBUG [RS_FLUSH_OPERATIONS-regionserver/037c58a19ddb:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=47 java.io.IOException: Unable to complete flush {ENCODED => 7c7f87a6124bfe2f4049ee7803e0f98c, NAME => 'TestAcidGuarantees,,1731751108989.7c7f87a6124bfe2f4049ee7803e0f98c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T09:58:45,256 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37877 {}] master.HMaster(4177): Remote procedure failed, pid=47 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 7c7f87a6124bfe2f4049ee7803e0f98c, NAME => 'TestAcidGuarantees,,1731751108989.7c7f87a6124bfe2f4049ee7803e0f98c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 7c7f87a6124bfe2f4049ee7803e0f98c, NAME => 'TestAcidGuarantees,,1731751108989.7c7f87a6124bfe2f4049ee7803e0f98c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T09:58:45,259 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/7c7f87a6124bfe2f4049ee7803e0f98c/.tmp/C/492eaebe25d74d938f56f67c6caf697f is 50, key is test_row_0/C:col10/1731751124326/Put/seqid=0 2024-11-16T09:58:45,265 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37591 is added to blk_1073742040_1216 (size=12301) 2024-11-16T09:58:45,409 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=37267 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=47 2024-11-16T09:58:45,409 DEBUG [RS_FLUSH_OPERATIONS-regionserver/037c58a19ddb:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731751108989.7c7f87a6124bfe2f4049ee7803e0f98c. 2024-11-16T09:58:45,410 DEBUG [RS_FLUSH_OPERATIONS-regionserver/037c58a19ddb:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731751108989.7c7f87a6124bfe2f4049ee7803e0f98c. as already flushing 2024-11-16T09:58:45,410 DEBUG [RS_FLUSH_OPERATIONS-regionserver/037c58a19ddb:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731751108989.7c7f87a6124bfe2f4049ee7803e0f98c. 2024-11-16T09:58:45,410 ERROR [RS_FLUSH_OPERATIONS-regionserver/037c58a19ddb:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] handler.RSProcedureHandler(58): pid=47 java.io.IOException: Unable to complete flush {ENCODED => 7c7f87a6124bfe2f4049ee7803e0f98c, NAME => 'TestAcidGuarantees,,1731751108989.7c7f87a6124bfe2f4049ee7803e0f98c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T09:58:45,410 DEBUG [RS_FLUSH_OPERATIONS-regionserver/037c58a19ddb:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=47 java.io.IOException: Unable to complete flush {ENCODED => 7c7f87a6124bfe2f4049ee7803e0f98c, NAME => 'TestAcidGuarantees,,1731751108989.7c7f87a6124bfe2f4049ee7803e0f98c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T09:58:45,411 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37877 {}] master.HMaster(4177): Remote procedure failed, pid=47 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 7c7f87a6124bfe2f4049ee7803e0f98c, NAME => 'TestAcidGuarantees,,1731751108989.7c7f87a6124bfe2f4049ee7803e0f98c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 7c7f87a6124bfe2f4049ee7803e0f98c, NAME => 'TestAcidGuarantees,,1731751108989.7c7f87a6124bfe2f4049ee7803e0f98c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T09:58:45,423 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37877 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=46 2024-11-16T09:58:45,503 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c7f87a6124bfe2f4049ee7803e0f98c, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T09:58:45,503 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] ipc.CallRunner(138): callId: 87 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52392 deadline: 1731751185502, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c7f87a6124bfe2f4049ee7803e0f98c, server=037c58a19ddb,37267,1731751077844 2024-11-16T09:58:45,503 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731751108989.7c7f87a6124bfe2f4049ee7803e0f98c., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 , the old value is region=TestAcidGuarantees,,1731751108989.7c7f87a6124bfe2f4049ee7803e0f98c., hostname=037c58a19ddb,37267,1731751077844, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c7f87a6124bfe2f4049ee7803e0f98c, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:58:45,503 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731751108989.7c7f87a6124bfe2f4049ee7803e0f98c., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c7f87a6124bfe2f4049ee7803e0f98c, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:58:45,503 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731751108989.7c7f87a6124bfe2f4049ee7803e0f98c., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 because the exception is null or not the one we care about 2024-11-16T09:58:45,515 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c7f87a6124bfe2f4049ee7803e0f98c, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T09:58:45,516 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] ipc.CallRunner(138): callId: 105 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52400 deadline: 1731751185513, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c7f87a6124bfe2f4049ee7803e0f98c, server=037c58a19ddb,37267,1731751077844 2024-11-16T09:58:45,516 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c7f87a6124bfe2f4049ee7803e0f98c, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T09:58:45,516 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] ipc.CallRunner(138): callId: 97 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52420 deadline: 1731751185513, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c7f87a6124bfe2f4049ee7803e0f98c, server=037c58a19ddb,37267,1731751077844 2024-11-16T09:58:45,516 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c7f87a6124bfe2f4049ee7803e0f98c, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T09:58:45,516 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] ipc.CallRunner(138): callId: 102 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52396 deadline: 1731751185513, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c7f87a6124bfe2f4049ee7803e0f98c, server=037c58a19ddb,37267,1731751077844 2024-11-16T09:58:45,517 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731751108989.7c7f87a6124bfe2f4049ee7803e0f98c., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 , the old value is region=TestAcidGuarantees,,1731751108989.7c7f87a6124bfe2f4049ee7803e0f98c., hostname=037c58a19ddb,37267,1731751077844, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c7f87a6124bfe2f4049ee7803e0f98c, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:58:45,517 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731751108989.7c7f87a6124bfe2f4049ee7803e0f98c., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 , the old value is region=TestAcidGuarantees,,1731751108989.7c7f87a6124bfe2f4049ee7803e0f98c., hostname=037c58a19ddb,37267,1731751077844, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c7f87a6124bfe2f4049ee7803e0f98c, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:58:45,517 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731751108989.7c7f87a6124bfe2f4049ee7803e0f98c., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 , the old value is region=TestAcidGuarantees,,1731751108989.7c7f87a6124bfe2f4049ee7803e0f98c., hostname=037c58a19ddb,37267,1731751077844, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c7f87a6124bfe2f4049ee7803e0f98c, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:58:45,517 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731751108989.7c7f87a6124bfe2f4049ee7803e0f98c., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c7f87a6124bfe2f4049ee7803e0f98c, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:58:45,517 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731751108989.7c7f87a6124bfe2f4049ee7803e0f98c., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c7f87a6124bfe2f4049ee7803e0f98c, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:58:45,517 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731751108989.7c7f87a6124bfe2f4049ee7803e0f98c., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c7f87a6124bfe2f4049ee7803e0f98c, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:58:45,517 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731751108989.7c7f87a6124bfe2f4049ee7803e0f98c., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 because the exception is null or not the one we care about 2024-11-16T09:58:45,517 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731751108989.7c7f87a6124bfe2f4049ee7803e0f98c., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 because the exception is null or not the one we care about 2024-11-16T09:58:45,517 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731751108989.7c7f87a6124bfe2f4049ee7803e0f98c., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 because the exception is null or not the one we care about 2024-11-16T09:58:45,563 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=37267 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=47 2024-11-16T09:58:45,564 DEBUG [RS_FLUSH_OPERATIONS-regionserver/037c58a19ddb:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731751108989.7c7f87a6124bfe2f4049ee7803e0f98c. 2024-11-16T09:58:45,564 DEBUG [RS_FLUSH_OPERATIONS-regionserver/037c58a19ddb:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731751108989.7c7f87a6124bfe2f4049ee7803e0f98c. as already flushing 2024-11-16T09:58:45,564 DEBUG [RS_FLUSH_OPERATIONS-regionserver/037c58a19ddb:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731751108989.7c7f87a6124bfe2f4049ee7803e0f98c. 2024-11-16T09:58:45,564 ERROR [RS_FLUSH_OPERATIONS-regionserver/037c58a19ddb:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] handler.RSProcedureHandler(58): pid=47 java.io.IOException: Unable to complete flush {ENCODED => 7c7f87a6124bfe2f4049ee7803e0f98c, NAME => 'TestAcidGuarantees,,1731751108989.7c7f87a6124bfe2f4049ee7803e0f98c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T09:58:45,564 DEBUG [RS_FLUSH_OPERATIONS-regionserver/037c58a19ddb:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=47 java.io.IOException: Unable to complete flush {ENCODED => 7c7f87a6124bfe2f4049ee7803e0f98c, NAME => 'TestAcidGuarantees,,1731751108989.7c7f87a6124bfe2f4049ee7803e0f98c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T09:58:45,565 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37877 {}] master.HMaster(4177): Remote procedure failed, pid=47 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 7c7f87a6124bfe2f4049ee7803e0f98c, NAME => 'TestAcidGuarantees,,1731751108989.7c7f87a6124bfe2f4049ee7803e0f98c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 7c7f87a6124bfe2f4049ee7803e0f98c, NAME => 'TestAcidGuarantees,,1731751108989.7c7f87a6124bfe2f4049ee7803e0f98c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T09:58:45,666 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=271 (bloomFilter=true), to=hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/7c7f87a6124bfe2f4049ee7803e0f98c/.tmp/C/492eaebe25d74d938f56f67c6caf697f 2024-11-16T09:58:45,676 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/7c7f87a6124bfe2f4049ee7803e0f98c/.tmp/A/36f2297c7adc4ecaaf1ca484ad85f45f as hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/7c7f87a6124bfe2f4049ee7803e0f98c/A/36f2297c7adc4ecaaf1ca484ad85f45f 2024-11-16T09:58:45,687 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/7c7f87a6124bfe2f4049ee7803e0f98c/A/36f2297c7adc4ecaaf1ca484ad85f45f, entries=200, sequenceid=271, filesize=39.0 K 2024-11-16T09:58:45,689 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/7c7f87a6124bfe2f4049ee7803e0f98c/.tmp/B/cdd6bdee112a48518295ab24f2e5e365 as hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/7c7f87a6124bfe2f4049ee7803e0f98c/B/cdd6bdee112a48518295ab24f2e5e365 2024-11-16T09:58:45,695 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/7c7f87a6124bfe2f4049ee7803e0f98c/B/cdd6bdee112a48518295ab24f2e5e365, entries=150, sequenceid=271, filesize=12.0 K 2024-11-16T09:58:45,697 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/7c7f87a6124bfe2f4049ee7803e0f98c/.tmp/C/492eaebe25d74d938f56f67c6caf697f as hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/7c7f87a6124bfe2f4049ee7803e0f98c/C/492eaebe25d74d938f56f67c6caf697f 2024-11-16T09:58:45,702 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/7c7f87a6124bfe2f4049ee7803e0f98c/C/492eaebe25d74d938f56f67c6caf697f, entries=150, sequenceid=271, filesize=12.0 K 2024-11-16T09:58:45,703 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~100.63 KB/103050, heapSize ~264.38 KB/270720, currentSize=100.63 KB/103050 for 7c7f87a6124bfe2f4049ee7803e0f98c in 1377ms, sequenceid=271, compaction requested=true 2024-11-16T09:58:45,703 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 7c7f87a6124bfe2f4049ee7803e0f98c: 2024-11-16T09:58:45,703 DEBUG [RS:0;037c58a19ddb:37267-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-16T09:58:45,705 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 7c7f87a6124bfe2f4049ee7803e0f98c:A, priority=-2147483648, current under compaction store size is 1 2024-11-16T09:58:45,705 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-16T09:58:45,705 DEBUG [RS:0;037c58a19ddb:37267-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-16T09:58:45,705 DEBUG [RS:0;037c58a19ddb:37267-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 142386 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-16T09:58:45,705 DEBUG [RS:0;037c58a19ddb:37267-shortCompactions-0 {}] regionserver.HStore(1541): 7c7f87a6124bfe2f4049ee7803e0f98c/A is initiating minor compaction (all files) 2024-11-16T09:58:45,705 INFO [RS:0;037c58a19ddb:37267-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 7c7f87a6124bfe2f4049ee7803e0f98c/A in TestAcidGuarantees,,1731751108989.7c7f87a6124bfe2f4049ee7803e0f98c. 2024-11-16T09:58:45,706 INFO [RS:0;037c58a19ddb:37267-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/7c7f87a6124bfe2f4049ee7803e0f98c/A/6b3a4c4820f0410ba008ad14ccf8af07, hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/7c7f87a6124bfe2f4049ee7803e0f98c/A/2049d15304474f38af50e9ff363d240f, hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/7c7f87a6124bfe2f4049ee7803e0f98c/A/15c9a113751c448ab4ea0ce9821776a1, hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/7c7f87a6124bfe2f4049ee7803e0f98c/A/36f2297c7adc4ecaaf1ca484ad85f45f] into tmpdir=hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/7c7f87a6124bfe2f4049ee7803e0f98c/.tmp, totalSize=139.0 K 2024-11-16T09:58:45,706 INFO [RS:0;037c58a19ddb:37267-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=12 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731751108989.7c7f87a6124bfe2f4049ee7803e0f98c. 2024-11-16T09:58:45,706 DEBUG [RS:0;037c58a19ddb:37267-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731751108989.7c7f87a6124bfe2f4049ee7803e0f98c. files: [hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/7c7f87a6124bfe2f4049ee7803e0f98c/A/6b3a4c4820f0410ba008ad14ccf8af07, hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/7c7f87a6124bfe2f4049ee7803e0f98c/A/2049d15304474f38af50e9ff363d240f, hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/7c7f87a6124bfe2f4049ee7803e0f98c/A/15c9a113751c448ab4ea0ce9821776a1, hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/7c7f87a6124bfe2f4049ee7803e0f98c/A/36f2297c7adc4ecaaf1ca484ad85f45f] 2024-11-16T09:58:45,707 DEBUG [RS:0;037c58a19ddb:37267-shortCompactions-0 {}] compactions.Compactor(225): Compacting 6b3a4c4820f0410ba008ad14ccf8af07, keycount=150, bloomtype=ROW, size=30.8 K, encoding=NONE, compression=NONE, seqNum=213, earliestPutTs=1731751121623 2024-11-16T09:58:45,707 DEBUG [RS:0;037c58a19ddb:37267-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 49232 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-16T09:58:45,707 DEBUG [RS:0;037c58a19ddb:37267-longCompactions-0 {}] regionserver.HStore(1541): 7c7f87a6124bfe2f4049ee7803e0f98c/B is initiating minor compaction (all files) 2024-11-16T09:58:45,707 INFO [RS:0;037c58a19ddb:37267-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 7c7f87a6124bfe2f4049ee7803e0f98c/B in TestAcidGuarantees,,1731751108989.7c7f87a6124bfe2f4049ee7803e0f98c. 2024-11-16T09:58:45,707 INFO [RS:0;037c58a19ddb:37267-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/7c7f87a6124bfe2f4049ee7803e0f98c/B/9989e7c7af994090af37ab514c81a8d1, hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/7c7f87a6124bfe2f4049ee7803e0f98c/B/a59127e94a214b4db7e66865fc9a2d7a, hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/7c7f87a6124bfe2f4049ee7803e0f98c/B/a45924af8cf74348bbdc6ab01952e6e7, hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/7c7f87a6124bfe2f4049ee7803e0f98c/B/cdd6bdee112a48518295ab24f2e5e365] into tmpdir=hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/7c7f87a6124bfe2f4049ee7803e0f98c/.tmp, totalSize=48.1 K 2024-11-16T09:58:45,707 DEBUG [RS:0;037c58a19ddb:37267-shortCompactions-0 {}] compactions.Compactor(225): Compacting 2049d15304474f38af50e9ff363d240f, keycount=200, bloomtype=ROW, size=38.8 K, encoding=NONE, compression=NONE, seqNum=229, earliestPutTs=1731751122310 2024-11-16T09:58:45,708 DEBUG [RS:0;037c58a19ddb:37267-longCompactions-0 {}] compactions.Compactor(225): Compacting 9989e7c7af994090af37ab514c81a8d1, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=213, earliestPutTs=1731751121623 2024-11-16T09:58:45,708 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 7c7f87a6124bfe2f4049ee7803e0f98c:B, priority=-2147483648, current under compaction store size is 2 2024-11-16T09:58:45,708 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-16T09:58:45,708 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 7c7f87a6124bfe2f4049ee7803e0f98c:C, priority=-2147483648, current under compaction store size is 3 2024-11-16T09:58:45,708 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-16T09:58:45,709 DEBUG [RS:0;037c58a19ddb:37267-shortCompactions-0 {}] compactions.Compactor(225): Compacting 15c9a113751c448ab4ea0ce9821776a1, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=252, earliestPutTs=1731751123004 2024-11-16T09:58:45,709 DEBUG [RS:0;037c58a19ddb:37267-longCompactions-0 {}] compactions.Compactor(225): Compacting a59127e94a214b4db7e66865fc9a2d7a, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=229, earliestPutTs=1731751122945 2024-11-16T09:58:45,709 DEBUG [RS:0;037c58a19ddb:37267-shortCompactions-0 {}] compactions.Compactor(225): Compacting 36f2297c7adc4ecaaf1ca484ad85f45f, keycount=200, bloomtype=ROW, size=39.0 K, encoding=NONE, compression=NONE, seqNum=271, earliestPutTs=1731751123693 2024-11-16T09:58:45,711 DEBUG [RS:0;037c58a19ddb:37267-longCompactions-0 {}] compactions.Compactor(225): Compacting a45924af8cf74348bbdc6ab01952e6e7, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=252, earliestPutTs=1731751123004 2024-11-16T09:58:45,712 DEBUG [RS:0;037c58a19ddb:37267-longCompactions-0 {}] compactions.Compactor(225): Compacting cdd6bdee112a48518295ab24f2e5e365, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=271, earliestPutTs=1731751123693 2024-11-16T09:58:45,717 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=37267 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=47 2024-11-16T09:58:45,718 DEBUG [RS_FLUSH_OPERATIONS-regionserver/037c58a19ddb:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731751108989.7c7f87a6124bfe2f4049ee7803e0f98c. 2024-11-16T09:58:45,718 INFO [RS_FLUSH_OPERATIONS-regionserver/037c58a19ddb:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HRegion(2902): Flushing 7c7f87a6124bfe2f4049ee7803e0f98c 3/3 column families, dataSize=100.63 KB heapSize=264.42 KB 2024-11-16T09:58:45,722 INFO [RS:0;037c58a19ddb:37267-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=7c7f87a6124bfe2f4049ee7803e0f98c] 2024-11-16T09:58:45,731 INFO [RS:0;037c58a19ddb:37267-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 7c7f87a6124bfe2f4049ee7803e0f98c#B#compaction#184 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-16T09:58:45,732 DEBUG [RS:0;037c58a19ddb:37267-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/7c7f87a6124bfe2f4049ee7803e0f98c/.tmp/B/82d565f1aaa9448aaf40a4bdcf5f4ddc is 50, key is test_row_0/B:col10/1731751124326/Put/seqid=0 2024-11-16T09:58:45,745 DEBUG [RS:0;037c58a19ddb:37267-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e202411165172e79f94674fe9a3db5135f3c19452_7c7f87a6124bfe2f4049ee7803e0f98c store=[table=TestAcidGuarantees family=A region=7c7f87a6124bfe2f4049ee7803e0f98c] 2024-11-16T09:58:45,748 DEBUG [RS:0;037c58a19ddb:37267-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e202411165172e79f94674fe9a3db5135f3c19452_7c7f87a6124bfe2f4049ee7803e0f98c, store=[table=TestAcidGuarantees family=A region=7c7f87a6124bfe2f4049ee7803e0f98c] 2024-11-16T09:58:45,748 DEBUG [RS:0;037c58a19ddb:37267-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411165172e79f94674fe9a3db5135f3c19452_7c7f87a6124bfe2f4049ee7803e0f98c because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=7c7f87a6124bfe2f4049ee7803e0f98c] 2024-11-16T09:58:45,751 DEBUG [RS_FLUSH_OPERATIONS-regionserver/037c58a19ddb:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411161d9574c60d754541b75a9ad2fe2e7796_7c7f87a6124bfe2f4049ee7803e0f98c is 50, key is test_row_0/A:col10/1731751124351/Put/seqid=0 2024-11-16T09:58:45,782 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37591 is added to blk_1073742041_1217 (size=12915) 2024-11-16T09:58:45,784 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37591 is added to blk_1073742043_1219 (size=12454) 2024-11-16T09:58:45,794 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37591 is added to blk_1073742042_1218 (size=4469) 2024-11-16T09:58:45,796 INFO [RS:0;037c58a19ddb:37267-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 7c7f87a6124bfe2f4049ee7803e0f98c#A#compaction#183 average throughput is 0.33 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-16T09:58:45,797 DEBUG [RS:0;037c58a19ddb:37267-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/7c7f87a6124bfe2f4049ee7803e0f98c/.tmp/A/cf911fd937c34d99a23ab70841f56bfe is 175, key is test_row_0/A:col10/1731751124326/Put/seqid=0 2024-11-16T09:58:45,799 DEBUG [RS:0;037c58a19ddb:37267-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/7c7f87a6124bfe2f4049ee7803e0f98c/.tmp/B/82d565f1aaa9448aaf40a4bdcf5f4ddc as hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/7c7f87a6124bfe2f4049ee7803e0f98c/B/82d565f1aaa9448aaf40a4bdcf5f4ddc 2024-11-16T09:58:45,807 INFO [RS:0;037c58a19ddb:37267-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 7c7f87a6124bfe2f4049ee7803e0f98c/B of 7c7f87a6124bfe2f4049ee7803e0f98c into 82d565f1aaa9448aaf40a4bdcf5f4ddc(size=12.6 K), total size for store is 12.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-16T09:58:45,807 DEBUG [RS:0;037c58a19ddb:37267-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 7c7f87a6124bfe2f4049ee7803e0f98c: 2024-11-16T09:58:45,807 INFO [RS:0;037c58a19ddb:37267-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731751108989.7c7f87a6124bfe2f4049ee7803e0f98c., storeName=7c7f87a6124bfe2f4049ee7803e0f98c/B, priority=12, startTime=1731751125705; duration=0sec 2024-11-16T09:58:45,807 DEBUG [RS:0;037c58a19ddb:37267-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-16T09:58:45,807 DEBUG [RS:0;037c58a19ddb:37267-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 7c7f87a6124bfe2f4049ee7803e0f98c:B 2024-11-16T09:58:45,807 DEBUG [RS:0;037c58a19ddb:37267-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-16T09:58:45,809 DEBUG [RS:0;037c58a19ddb:37267-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 49232 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-16T09:58:45,809 DEBUG [RS:0;037c58a19ddb:37267-longCompactions-0 {}] regionserver.HStore(1541): 7c7f87a6124bfe2f4049ee7803e0f98c/C is initiating minor compaction (all files) 2024-11-16T09:58:45,809 INFO [RS:0;037c58a19ddb:37267-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 7c7f87a6124bfe2f4049ee7803e0f98c/C in TestAcidGuarantees,,1731751108989.7c7f87a6124bfe2f4049ee7803e0f98c. 2024-11-16T09:58:45,809 INFO [RS:0;037c58a19ddb:37267-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/7c7f87a6124bfe2f4049ee7803e0f98c/C/4890337659474a46a61bf83aba890fb0, hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/7c7f87a6124bfe2f4049ee7803e0f98c/C/e1471e6f23124679a598ae6112b8b618, hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/7c7f87a6124bfe2f4049ee7803e0f98c/C/3064c932d4fc4dc49cae309d5c3bdfbe, hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/7c7f87a6124bfe2f4049ee7803e0f98c/C/492eaebe25d74d938f56f67c6caf697f] into tmpdir=hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/7c7f87a6124bfe2f4049ee7803e0f98c/.tmp, totalSize=48.1 K 2024-11-16T09:58:45,810 DEBUG [RS:0;037c58a19ddb:37267-longCompactions-0 {}] compactions.Compactor(225): Compacting 4890337659474a46a61bf83aba890fb0, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=213, earliestPutTs=1731751121623 2024-11-16T09:58:45,810 DEBUG [RS:0;037c58a19ddb:37267-longCompactions-0 {}] compactions.Compactor(225): Compacting e1471e6f23124679a598ae6112b8b618, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=229, earliestPutTs=1731751122945 2024-11-16T09:58:45,810 DEBUG [RS:0;037c58a19ddb:37267-longCompactions-0 {}] compactions.Compactor(225): Compacting 3064c932d4fc4dc49cae309d5c3bdfbe, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=252, earliestPutTs=1731751123004 2024-11-16T09:58:45,811 DEBUG [RS:0;037c58a19ddb:37267-longCompactions-0 {}] compactions.Compactor(225): Compacting 492eaebe25d74d938f56f67c6caf697f, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=271, earliestPutTs=1731751123693 2024-11-16T09:58:45,813 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37591 is added to blk_1073742044_1220 (size=31869) 2024-11-16T09:58:45,824 DEBUG [RS:0;037c58a19ddb:37267-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/7c7f87a6124bfe2f4049ee7803e0f98c/.tmp/A/cf911fd937c34d99a23ab70841f56bfe as hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/7c7f87a6124bfe2f4049ee7803e0f98c/A/cf911fd937c34d99a23ab70841f56bfe 2024-11-16T09:58:45,830 INFO [RS:0;037c58a19ddb:37267-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 7c7f87a6124bfe2f4049ee7803e0f98c#C#compaction#186 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-16T09:58:45,831 DEBUG [RS:0;037c58a19ddb:37267-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/7c7f87a6124bfe2f4049ee7803e0f98c/.tmp/C/c9f68351e8204e01acd0f22f12b4cc64 is 50, key is test_row_0/C:col10/1731751124326/Put/seqid=0 2024-11-16T09:58:45,837 INFO [RS:0;037c58a19ddb:37267-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 7c7f87a6124bfe2f4049ee7803e0f98c/A of 7c7f87a6124bfe2f4049ee7803e0f98c into cf911fd937c34d99a23ab70841f56bfe(size=31.1 K), total size for store is 31.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-16T09:58:45,837 DEBUG [RS:0;037c58a19ddb:37267-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 7c7f87a6124bfe2f4049ee7803e0f98c: 2024-11-16T09:58:45,837 INFO [RS:0;037c58a19ddb:37267-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731751108989.7c7f87a6124bfe2f4049ee7803e0f98c., storeName=7c7f87a6124bfe2f4049ee7803e0f98c/A, priority=12, startTime=1731751125703; duration=0sec 2024-11-16T09:58:45,838 DEBUG [RS:0;037c58a19ddb:37267-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-16T09:58:45,838 DEBUG [RS:0;037c58a19ddb:37267-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 7c7f87a6124bfe2f4049ee7803e0f98c:A 2024-11-16T09:58:45,856 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37591 is added to blk_1073742045_1221 (size=12915) 2024-11-16T09:58:45,865 DEBUG [RS:0;037c58a19ddb:37267-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/7c7f87a6124bfe2f4049ee7803e0f98c/.tmp/C/c9f68351e8204e01acd0f22f12b4cc64 as hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/7c7f87a6124bfe2f4049ee7803e0f98c/C/c9f68351e8204e01acd0f22f12b4cc64 2024-11-16T09:58:45,873 INFO [RS:0;037c58a19ddb:37267-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 7c7f87a6124bfe2f4049ee7803e0f98c/C of 7c7f87a6124bfe2f4049ee7803e0f98c into c9f68351e8204e01acd0f22f12b4cc64(size=12.6 K), total size for store is 12.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-16T09:58:45,873 DEBUG [RS:0;037c58a19ddb:37267-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 7c7f87a6124bfe2f4049ee7803e0f98c: 2024-11-16T09:58:45,873 INFO [RS:0;037c58a19ddb:37267-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731751108989.7c7f87a6124bfe2f4049ee7803e0f98c., storeName=7c7f87a6124bfe2f4049ee7803e0f98c/C, priority=12, startTime=1731751125708; duration=0sec 2024-11-16T09:58:45,874 DEBUG [RS:0;037c58a19ddb:37267-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-16T09:58:45,874 DEBUG [RS:0;037c58a19ddb:37267-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 7c7f87a6124bfe2f4049ee7803e0f98c:C 2024-11-16T09:58:45,933 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37877 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=46 2024-11-16T09:58:46,193 DEBUG [RS_FLUSH_OPERATIONS-regionserver/037c58a19ddb:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T09:58:46,198 INFO [RS_FLUSH_OPERATIONS-regionserver/037c58a19ddb:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411161d9574c60d754541b75a9ad2fe2e7796_7c7f87a6124bfe2f4049ee7803e0f98c to hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411161d9574c60d754541b75a9ad2fe2e7796_7c7f87a6124bfe2f4049ee7803e0f98c 2024-11-16T09:58:46,200 DEBUG [RS_FLUSH_OPERATIONS-regionserver/037c58a19ddb:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/7c7f87a6124bfe2f4049ee7803e0f98c/.tmp/A/5ea7fb182fe64f0b9234c748529c1dec, store: [table=TestAcidGuarantees family=A region=7c7f87a6124bfe2f4049ee7803e0f98c] 2024-11-16T09:58:46,200 DEBUG [RS_FLUSH_OPERATIONS-regionserver/037c58a19ddb:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/7c7f87a6124bfe2f4049ee7803e0f98c/.tmp/A/5ea7fb182fe64f0b9234c748529c1dec is 175, key is test_row_0/A:col10/1731751124351/Put/seqid=0 2024-11-16T09:58:46,239 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37591 is added to blk_1073742046_1222 (size=31255) 2024-11-16T09:58:46,385 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] regionserver.HRegion(8855): Flush requested on 7c7f87a6124bfe2f4049ee7803e0f98c 2024-11-16T09:58:46,385 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731751108989.7c7f87a6124bfe2f4049ee7803e0f98c. as already flushing 2024-11-16T09:58:46,452 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c7f87a6124bfe2f4049ee7803e0f98c, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T09:58:46,453 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52428 deadline: 1731751186448, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c7f87a6124bfe2f4049ee7803e0f98c, server=037c58a19ddb,37267,1731751077844 2024-11-16T09:58:46,453 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731751108989.7c7f87a6124bfe2f4049ee7803e0f98c., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 , the old value is region=TestAcidGuarantees,,1731751108989.7c7f87a6124bfe2f4049ee7803e0f98c., hostname=037c58a19ddb,37267,1731751077844, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c7f87a6124bfe2f4049ee7803e0f98c, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:58:46,453 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731751108989.7c7f87a6124bfe2f4049ee7803e0f98c., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c7f87a6124bfe2f4049ee7803e0f98c, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:58:46,453 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731751108989.7c7f87a6124bfe2f4049ee7803e0f98c., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 because the exception is null or not the one we care about 2024-11-16T09:58:46,524 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c7f87a6124bfe2f4049ee7803e0f98c, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T09:58:46,524 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] ipc.CallRunner(138): callId: 88 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52392 deadline: 1731751186522, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c7f87a6124bfe2f4049ee7803e0f98c, server=037c58a19ddb,37267,1731751077844 2024-11-16T09:58:46,524 WARN [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRpcRetryingCaller(177): Call to 037c58a19ddb,37267,1731751077844 for 'test_row_2' in 7c7f87a6124bfe2f4049ee7803e0f98c of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2166 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c7f87a6124bfe2f4049ee7803e0f98c, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor44.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c7f87a6124bfe2f4049ee7803e0f98c, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-16T09:58:46,525 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731751108989.7c7f87a6124bfe2f4049ee7803e0f98c., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 , the old value is region=TestAcidGuarantees,,1731751108989.7c7f87a6124bfe2f4049ee7803e0f98c., hostname=037c58a19ddb,37267,1731751077844, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c7f87a6124bfe2f4049ee7803e0f98c, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:58:46,525 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731751108989.7c7f87a6124bfe2f4049ee7803e0f98c., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c7f87a6124bfe2f4049ee7803e0f98c, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:58:46,525 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731751108989.7c7f87a6124bfe2f4049ee7803e0f98c., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 because the exception is null or not the one we care about 2024-11-16T09:58:46,536 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c7f87a6124bfe2f4049ee7803e0f98c, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T09:58:46,536 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] ipc.CallRunner(138): callId: 103 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52396 deadline: 1731751186533, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c7f87a6124bfe2f4049ee7803e0f98c, server=037c58a19ddb,37267,1731751077844 2024-11-16T09:58:46,537 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c7f87a6124bfe2f4049ee7803e0f98c, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T09:58:46,537 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] ipc.CallRunner(138): callId: 98 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52420 deadline: 1731751186533, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c7f87a6124bfe2f4049ee7803e0f98c, server=037c58a19ddb,37267,1731751077844 2024-11-16T09:58:46,537 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c7f87a6124bfe2f4049ee7803e0f98c, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T09:58:46,537 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] ipc.CallRunner(138): callId: 106 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52400 deadline: 1731751186533, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c7f87a6124bfe2f4049ee7803e0f98c, server=037c58a19ddb,37267,1731751077844 2024-11-16T09:58:46,537 WARN [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRpcRetryingCaller(177): Call to 037c58a19ddb,37267,1731751077844 for 'test_row_2' in 7c7f87a6124bfe2f4049ee7803e0f98c of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2183 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c7f87a6124bfe2f4049ee7803e0f98c, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor44.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c7f87a6124bfe2f4049ee7803e0f98c, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-16T09:58:46,537 WARN [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRpcRetryingCaller(177): Call to 037c58a19ddb,37267,1731751077844 for 'test_row_2' in 7c7f87a6124bfe2f4049ee7803e0f98c of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2181 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c7f87a6124bfe2f4049ee7803e0f98c, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor44.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c7f87a6124bfe2f4049ee7803e0f98c, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-16T09:58:46,537 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731751108989.7c7f87a6124bfe2f4049ee7803e0f98c., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 , the old value is region=TestAcidGuarantees,,1731751108989.7c7f87a6124bfe2f4049ee7803e0f98c., hostname=037c58a19ddb,37267,1731751077844, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c7f87a6124bfe2f4049ee7803e0f98c, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:58:46,538 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731751108989.7c7f87a6124bfe2f4049ee7803e0f98c., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c7f87a6124bfe2f4049ee7803e0f98c, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:58:46,538 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731751108989.7c7f87a6124bfe2f4049ee7803e0f98c., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 , the old value is region=TestAcidGuarantees,,1731751108989.7c7f87a6124bfe2f4049ee7803e0f98c., hostname=037c58a19ddb,37267,1731751077844, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c7f87a6124bfe2f4049ee7803e0f98c, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:58:46,538 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731751108989.7c7f87a6124bfe2f4049ee7803e0f98c., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 because the exception is null or not the one we care about 2024-11-16T09:58:46,538 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731751108989.7c7f87a6124bfe2f4049ee7803e0f98c., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c7f87a6124bfe2f4049ee7803e0f98c, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:58:46,538 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731751108989.7c7f87a6124bfe2f4049ee7803e0f98c., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 because the exception is null or not the one we care about 2024-11-16T09:58:46,538 WARN [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRpcRetryingCaller(177): Call to 037c58a19ddb,37267,1731751077844 for 'test_row_1' in 7c7f87a6124bfe2f4049ee7803e0f98c of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2176 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c7f87a6124bfe2f4049ee7803e0f98c, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor44.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c7f87a6124bfe2f4049ee7803e0f98c, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-16T09:58:46,538 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731751108989.7c7f87a6124bfe2f4049ee7803e0f98c., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 , the old value is region=TestAcidGuarantees,,1731751108989.7c7f87a6124bfe2f4049ee7803e0f98c., hostname=037c58a19ddb,37267,1731751077844, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c7f87a6124bfe2f4049ee7803e0f98c, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:58:46,539 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731751108989.7c7f87a6124bfe2f4049ee7803e0f98c., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c7f87a6124bfe2f4049ee7803e0f98c, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:58:46,539 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731751108989.7c7f87a6124bfe2f4049ee7803e0f98c., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 because the exception is null or not the one we care about 2024-11-16T09:58:46,565 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c7f87a6124bfe2f4049ee7803e0f98c, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T09:58:46,565 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52428 deadline: 1731751186563, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c7f87a6124bfe2f4049ee7803e0f98c, server=037c58a19ddb,37267,1731751077844 2024-11-16T09:58:46,566 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731751108989.7c7f87a6124bfe2f4049ee7803e0f98c., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 , the old value is region=TestAcidGuarantees,,1731751108989.7c7f87a6124bfe2f4049ee7803e0f98c., hostname=037c58a19ddb,37267,1731751077844, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c7f87a6124bfe2f4049ee7803e0f98c, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:58:46,566 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731751108989.7c7f87a6124bfe2f4049ee7803e0f98c., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c7f87a6124bfe2f4049ee7803e0f98c, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:58:46,566 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731751108989.7c7f87a6124bfe2f4049ee7803e0f98c., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 because the exception is null or not the one we care about 2024-11-16T09:58:46,640 INFO [RS_FLUSH_OPERATIONS-regionserver/037c58a19ddb:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=289, memsize=33.5 K, hasBloomFilter=true, into tmp file hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/7c7f87a6124bfe2f4049ee7803e0f98c/.tmp/A/5ea7fb182fe64f0b9234c748529c1dec 2024-11-16T09:58:46,649 DEBUG [RS_FLUSH_OPERATIONS-regionserver/037c58a19ddb:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/7c7f87a6124bfe2f4049ee7803e0f98c/.tmp/B/d5f199b181be423483c2bb9c67b0a823 is 50, key is test_row_0/B:col10/1731751124351/Put/seqid=0 2024-11-16T09:58:46,653 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37591 is added to blk_1073742047_1223 (size=12301) 2024-11-16T09:58:46,774 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c7f87a6124bfe2f4049ee7803e0f98c, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T09:58:46,774 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52428 deadline: 1731751186773, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c7f87a6124bfe2f4049ee7803e0f98c, server=037c58a19ddb,37267,1731751077844 2024-11-16T09:58:46,774 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731751108989.7c7f87a6124bfe2f4049ee7803e0f98c., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 , the old value is region=TestAcidGuarantees,,1731751108989.7c7f87a6124bfe2f4049ee7803e0f98c., hostname=037c58a19ddb,37267,1731751077844, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c7f87a6124bfe2f4049ee7803e0f98c, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:58:46,774 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731751108989.7c7f87a6124bfe2f4049ee7803e0f98c., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c7f87a6124bfe2f4049ee7803e0f98c, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:58:46,774 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731751108989.7c7f87a6124bfe2f4049ee7803e0f98c., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 because the exception is null or not the one we care about 2024-11-16T09:58:46,943 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37877 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=46 2024-11-16T09:58:47,054 INFO [RS_FLUSH_OPERATIONS-regionserver/037c58a19ddb:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=289 (bloomFilter=true), to=hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/7c7f87a6124bfe2f4049ee7803e0f98c/.tmp/B/d5f199b181be423483c2bb9c67b0a823 2024-11-16T09:58:47,062 DEBUG [RS_FLUSH_OPERATIONS-regionserver/037c58a19ddb:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/7c7f87a6124bfe2f4049ee7803e0f98c/.tmp/C/c8cbef2eeb3749d4b5fe5998814b7135 is 50, key is test_row_0/C:col10/1731751124351/Put/seqid=0 2024-11-16T09:58:47,066 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37591 is added to blk_1073742048_1224 (size=12301) 2024-11-16T09:58:47,085 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c7f87a6124bfe2f4049ee7803e0f98c, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T09:58:47,085 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52428 deadline: 1731751187083, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c7f87a6124bfe2f4049ee7803e0f98c, server=037c58a19ddb,37267,1731751077844 2024-11-16T09:58:47,086 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731751108989.7c7f87a6124bfe2f4049ee7803e0f98c., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 , the old value is region=TestAcidGuarantees,,1731751108989.7c7f87a6124bfe2f4049ee7803e0f98c., hostname=037c58a19ddb,37267,1731751077844, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c7f87a6124bfe2f4049ee7803e0f98c, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:58:47,086 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731751108989.7c7f87a6124bfe2f4049ee7803e0f98c., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c7f87a6124bfe2f4049ee7803e0f98c, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:58:47,086 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731751108989.7c7f87a6124bfe2f4049ee7803e0f98c., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 because the exception is null or not the one we care about 2024-11-16T09:58:47,467 INFO [RS_FLUSH_OPERATIONS-regionserver/037c58a19ddb:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=289 (bloomFilter=true), to=hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/7c7f87a6124bfe2f4049ee7803e0f98c/.tmp/C/c8cbef2eeb3749d4b5fe5998814b7135 2024-11-16T09:58:47,472 DEBUG [RS_FLUSH_OPERATIONS-regionserver/037c58a19ddb:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/7c7f87a6124bfe2f4049ee7803e0f98c/.tmp/A/5ea7fb182fe64f0b9234c748529c1dec as hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/7c7f87a6124bfe2f4049ee7803e0f98c/A/5ea7fb182fe64f0b9234c748529c1dec 2024-11-16T09:58:47,477 INFO [RS_FLUSH_OPERATIONS-regionserver/037c58a19ddb:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/7c7f87a6124bfe2f4049ee7803e0f98c/A/5ea7fb182fe64f0b9234c748529c1dec, entries=150, sequenceid=289, filesize=30.5 K 2024-11-16T09:58:47,478 DEBUG [RS_FLUSH_OPERATIONS-regionserver/037c58a19ddb:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/7c7f87a6124bfe2f4049ee7803e0f98c/.tmp/B/d5f199b181be423483c2bb9c67b0a823 as hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/7c7f87a6124bfe2f4049ee7803e0f98c/B/d5f199b181be423483c2bb9c67b0a823 2024-11-16T09:58:47,482 INFO [RS_FLUSH_OPERATIONS-regionserver/037c58a19ddb:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/7c7f87a6124bfe2f4049ee7803e0f98c/B/d5f199b181be423483c2bb9c67b0a823, entries=150, sequenceid=289, filesize=12.0 K 2024-11-16T09:58:47,483 DEBUG [RS_FLUSH_OPERATIONS-regionserver/037c58a19ddb:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/7c7f87a6124bfe2f4049ee7803e0f98c/.tmp/C/c8cbef2eeb3749d4b5fe5998814b7135 as hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/7c7f87a6124bfe2f4049ee7803e0f98c/C/c8cbef2eeb3749d4b5fe5998814b7135 2024-11-16T09:58:47,487 INFO [RS_FLUSH_OPERATIONS-regionserver/037c58a19ddb:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/7c7f87a6124bfe2f4049ee7803e0f98c/C/c8cbef2eeb3749d4b5fe5998814b7135, entries=150, sequenceid=289, filesize=12.0 K 2024-11-16T09:58:47,488 INFO [RS_FLUSH_OPERATIONS-regionserver/037c58a19ddb:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HRegion(3140): Finished flush of dataSize ~100.63 KB/103050, heapSize ~264.38 KB/270720, currentSize=100.63 KB/103050 for 7c7f87a6124bfe2f4049ee7803e0f98c in 1770ms, sequenceid=289, compaction requested=false 2024-11-16T09:58:47,488 DEBUG [RS_FLUSH_OPERATIONS-regionserver/037c58a19ddb:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HRegion(2603): Flush status journal for 7c7f87a6124bfe2f4049ee7803e0f98c: 2024-11-16T09:58:47,488 DEBUG [RS_FLUSH_OPERATIONS-regionserver/037c58a19ddb:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731751108989.7c7f87a6124bfe2f4049ee7803e0f98c. 2024-11-16T09:58:47,488 DEBUG [RS_FLUSH_OPERATIONS-regionserver/037c58a19ddb:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=47 2024-11-16T09:58:47,488 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37877 {}] master.HMaster(4169): Remote procedure done, pid=47 2024-11-16T09:58:47,491 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=47, resume processing ppid=46 2024-11-16T09:58:47,491 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=47, ppid=46, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.6930 sec 2024-11-16T09:58:47,492 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=46, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=46, table=TestAcidGuarantees in 2.6980 sec 2024-11-16T09:58:47,594 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] regionserver.HRegion(8855): Flush requested on 7c7f87a6124bfe2f4049ee7803e0f98c 2024-11-16T09:58:47,595 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-16T09:58:47,595 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 7c7f87a6124bfe2f4049ee7803e0f98c 3/3 column families, dataSize=107.34 KB heapSize=282 KB 2024-11-16T09:58:47,601 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241116f064654b48554817935f71a3f7529068_7c7f87a6124bfe2f4049ee7803e0f98c is 50, key is test_row_0/A:col10/1731751127593/Put/seqid=0 2024-11-16T09:58:47,605 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37591 is added to blk_1073742049_1225 (size=14994) 2024-11-16T09:58:47,664 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c7f87a6124bfe2f4049ee7803e0f98c, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T09:58:47,664 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] ipc.CallRunner(138): callId: 78 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52428 deadline: 1731751187664, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c7f87a6124bfe2f4049ee7803e0f98c, server=037c58a19ddb,37267,1731751077844 2024-11-16T09:58:47,664 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731751108989.7c7f87a6124bfe2f4049ee7803e0f98c., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 , the old value is region=TestAcidGuarantees,,1731751108989.7c7f87a6124bfe2f4049ee7803e0f98c., hostname=037c58a19ddb,37267,1731751077844, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c7f87a6124bfe2f4049ee7803e0f98c, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:58:47,665 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731751108989.7c7f87a6124bfe2f4049ee7803e0f98c., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c7f87a6124bfe2f4049ee7803e0f98c, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:58:47,665 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731751108989.7c7f87a6124bfe2f4049ee7803e0f98c., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 because the exception is null or not the one we care about 2024-11-16T09:58:47,774 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c7f87a6124bfe2f4049ee7803e0f98c, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T09:58:47,775 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] ipc.CallRunner(138): callId: 79 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52428 deadline: 1731751187773, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c7f87a6124bfe2f4049ee7803e0f98c, server=037c58a19ddb,37267,1731751077844 2024-11-16T09:58:47,775 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731751108989.7c7f87a6124bfe2f4049ee7803e0f98c., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 , the old value is region=TestAcidGuarantees,,1731751108989.7c7f87a6124bfe2f4049ee7803e0f98c., hostname=037c58a19ddb,37267,1731751077844, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c7f87a6124bfe2f4049ee7803e0f98c, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:58:47,775 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731751108989.7c7f87a6124bfe2f4049ee7803e0f98c., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c7f87a6124bfe2f4049ee7803e0f98c, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:58:47,775 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731751108989.7c7f87a6124bfe2f4049ee7803e0f98c., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 because the exception is null or not the one we care about 2024-11-16T09:58:47,983 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c7f87a6124bfe2f4049ee7803e0f98c, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T09:58:47,984 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] ipc.CallRunner(138): callId: 80 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52428 deadline: 1731751187983, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c7f87a6124bfe2f4049ee7803e0f98c, server=037c58a19ddb,37267,1731751077844 2024-11-16T09:58:47,984 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731751108989.7c7f87a6124bfe2f4049ee7803e0f98c., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 , the old value is region=TestAcidGuarantees,,1731751108989.7c7f87a6124bfe2f4049ee7803e0f98c., hostname=037c58a19ddb,37267,1731751077844, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c7f87a6124bfe2f4049ee7803e0f98c, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:58:47,984 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731751108989.7c7f87a6124bfe2f4049ee7803e0f98c., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c7f87a6124bfe2f4049ee7803e0f98c, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:58:47,984 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731751108989.7c7f87a6124bfe2f4049ee7803e0f98c., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 because the exception is null or not the one we care about 2024-11-16T09:58:48,006 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T09:58:48,010 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241116f064654b48554817935f71a3f7529068_7c7f87a6124bfe2f4049ee7803e0f98c to hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241116f064654b48554817935f71a3f7529068_7c7f87a6124bfe2f4049ee7803e0f98c 2024-11-16T09:58:48,011 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/7c7f87a6124bfe2f4049ee7803e0f98c/.tmp/A/34e9400bce784c0fa95be143ab8eefe9, store: [table=TestAcidGuarantees family=A region=7c7f87a6124bfe2f4049ee7803e0f98c] 2024-11-16T09:58:48,012 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/7c7f87a6124bfe2f4049ee7803e0f98c/.tmp/A/34e9400bce784c0fa95be143ab8eefe9 is 175, key is test_row_0/A:col10/1731751127593/Put/seqid=0 2024-11-16T09:58:48,016 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37591 is added to blk_1073742050_1226 (size=39949) 2024-11-16T09:58:48,293 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c7f87a6124bfe2f4049ee7803e0f98c, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T09:58:48,293 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] ipc.CallRunner(138): callId: 81 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52428 deadline: 1731751188293, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c7f87a6124bfe2f4049ee7803e0f98c, server=037c58a19ddb,37267,1731751077844 2024-11-16T09:58:48,294 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731751108989.7c7f87a6124bfe2f4049ee7803e0f98c., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 , the old value is region=TestAcidGuarantees,,1731751108989.7c7f87a6124bfe2f4049ee7803e0f98c., hostname=037c58a19ddb,37267,1731751077844, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c7f87a6124bfe2f4049ee7803e0f98c, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:58:48,294 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731751108989.7c7f87a6124bfe2f4049ee7803e0f98c., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c7f87a6124bfe2f4049ee7803e0f98c, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:58:48,294 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731751108989.7c7f87a6124bfe2f4049ee7803e0f98c., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 because the exception is null or not the one we care about 2024-11-16T09:58:48,417 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=311, memsize=35.8 K, hasBloomFilter=true, into tmp file hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/7c7f87a6124bfe2f4049ee7803e0f98c/.tmp/A/34e9400bce784c0fa95be143ab8eefe9 2024-11-16T09:58:48,426 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/7c7f87a6124bfe2f4049ee7803e0f98c/.tmp/B/724e91f5d6c845aeaf175fa51a5f0e39 is 50, key is test_row_0/B:col10/1731751127593/Put/seqid=0 2024-11-16T09:58:48,430 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37591 is added to blk_1073742051_1227 (size=12301) 2024-11-16T09:58:48,431 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=311 (bloomFilter=true), to=hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/7c7f87a6124bfe2f4049ee7803e0f98c/.tmp/B/724e91f5d6c845aeaf175fa51a5f0e39 2024-11-16T09:58:48,439 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/7c7f87a6124bfe2f4049ee7803e0f98c/.tmp/C/9c7d3a5858cc46d2aeb57bf167db023f is 50, key is test_row_0/C:col10/1731751127593/Put/seqid=0 2024-11-16T09:58:48,442 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37591 is added to blk_1073742052_1228 (size=12301) 2024-11-16T09:58:48,546 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c7f87a6124bfe2f4049ee7803e0f98c, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T09:58:48,547 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] ipc.CallRunner(138): callId: 89 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52392 deadline: 1731751188543, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c7f87a6124bfe2f4049ee7803e0f98c, server=037c58a19ddb,37267,1731751077844 2024-11-16T09:58:48,547 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c7f87a6124bfe2f4049ee7803e0f98c, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T09:58:48,547 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] ipc.CallRunner(138): callId: 107 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52400 deadline: 1731751188543, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c7f87a6124bfe2f4049ee7803e0f98c, server=037c58a19ddb,37267,1731751077844 2024-11-16T09:58:48,547 WARN [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRpcRetryingCaller(177): Call to 037c58a19ddb,37267,1731751077844 for 'test_row_2' in 7c7f87a6124bfe2f4049ee7803e0f98c of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4189 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c7f87a6124bfe2f4049ee7803e0f98c, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor44.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c7f87a6124bfe2f4049ee7803e0f98c, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-16T09:58:48,548 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731751108989.7c7f87a6124bfe2f4049ee7803e0f98c., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 , the old value is region=TestAcidGuarantees,,1731751108989.7c7f87a6124bfe2f4049ee7803e0f98c., hostname=037c58a19ddb,37267,1731751077844, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c7f87a6124bfe2f4049ee7803e0f98c, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:58:48,548 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731751108989.7c7f87a6124bfe2f4049ee7803e0f98c., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c7f87a6124bfe2f4049ee7803e0f98c, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:58:48,548 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731751108989.7c7f87a6124bfe2f4049ee7803e0f98c., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 because the exception is null or not the one we care about 2024-11-16T09:58:48,547 WARN [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRpcRetryingCaller(177): Call to 037c58a19ddb,37267,1731751077844 for 'test_row_1' in 7c7f87a6124bfe2f4049ee7803e0f98c of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4186 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c7f87a6124bfe2f4049ee7803e0f98c, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor44.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c7f87a6124bfe2f4049ee7803e0f98c, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-16T09:58:48,548 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731751108989.7c7f87a6124bfe2f4049ee7803e0f98c., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 , the old value is region=TestAcidGuarantees,,1731751108989.7c7f87a6124bfe2f4049ee7803e0f98c., hostname=037c58a19ddb,37267,1731751077844, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c7f87a6124bfe2f4049ee7803e0f98c, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:58:48,548 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731751108989.7c7f87a6124bfe2f4049ee7803e0f98c., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c7f87a6124bfe2f4049ee7803e0f98c, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:58:48,548 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731751108989.7c7f87a6124bfe2f4049ee7803e0f98c., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 because the exception is null or not the one we care about 2024-11-16T09:58:48,554 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c7f87a6124bfe2f4049ee7803e0f98c, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T09:58:48,555 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] ipc.CallRunner(138): callId: 104 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52396 deadline: 1731751188552, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c7f87a6124bfe2f4049ee7803e0f98c, server=037c58a19ddb,37267,1731751077844 2024-11-16T09:58:48,555 WARN [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRpcRetryingCaller(177): Call to 037c58a19ddb,37267,1731751077844 for 'test_row_2' in 7c7f87a6124bfe2f4049ee7803e0f98c of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4201 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c7f87a6124bfe2f4049ee7803e0f98c, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor44.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c7f87a6124bfe2f4049ee7803e0f98c, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-16T09:58:48,555 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731751108989.7c7f87a6124bfe2f4049ee7803e0f98c., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 , the old value is region=TestAcidGuarantees,,1731751108989.7c7f87a6124bfe2f4049ee7803e0f98c., hostname=037c58a19ddb,37267,1731751077844, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c7f87a6124bfe2f4049ee7803e0f98c, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:58:48,555 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731751108989.7c7f87a6124bfe2f4049ee7803e0f98c., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c7f87a6124bfe2f4049ee7803e0f98c, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:58:48,555 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731751108989.7c7f87a6124bfe2f4049ee7803e0f98c., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 because the exception is null or not the one we care about 2024-11-16T09:58:48,564 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c7f87a6124bfe2f4049ee7803e0f98c, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T09:58:48,564 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] ipc.CallRunner(138): callId: 99 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52420 deadline: 1731751188563, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c7f87a6124bfe2f4049ee7803e0f98c, server=037c58a19ddb,37267,1731751077844 2024-11-16T09:58:48,565 WARN [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRpcRetryingCaller(177): Call to 037c58a19ddb,37267,1731751077844 for 'test_row_2' in 7c7f87a6124bfe2f4049ee7803e0f98c of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4209 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c7f87a6124bfe2f4049ee7803e0f98c, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor44.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c7f87a6124bfe2f4049ee7803e0f98c, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-16T09:58:48,565 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731751108989.7c7f87a6124bfe2f4049ee7803e0f98c., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 , the old value is region=TestAcidGuarantees,,1731751108989.7c7f87a6124bfe2f4049ee7803e0f98c., hostname=037c58a19ddb,37267,1731751077844, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c7f87a6124bfe2f4049ee7803e0f98c, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:58:48,565 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731751108989.7c7f87a6124bfe2f4049ee7803e0f98c., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c7f87a6124bfe2f4049ee7803e0f98c, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:58:48,565 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731751108989.7c7f87a6124bfe2f4049ee7803e0f98c., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 because the exception is null or not the one we care about 2024-11-16T09:58:48,807 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c7f87a6124bfe2f4049ee7803e0f98c, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T09:58:48,807 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] ipc.CallRunner(138): callId: 82 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52428 deadline: 1731751188802, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c7f87a6124bfe2f4049ee7803e0f98c, server=037c58a19ddb,37267,1731751077844 2024-11-16T09:58:48,807 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731751108989.7c7f87a6124bfe2f4049ee7803e0f98c., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 , the old value is region=TestAcidGuarantees,,1731751108989.7c7f87a6124bfe2f4049ee7803e0f98c., hostname=037c58a19ddb,37267,1731751077844, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c7f87a6124bfe2f4049ee7803e0f98c, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:58:48,807 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731751108989.7c7f87a6124bfe2f4049ee7803e0f98c., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c7f87a6124bfe2f4049ee7803e0f98c, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:58:48,807 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731751108989.7c7f87a6124bfe2f4049ee7803e0f98c., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 because the exception is null or not the one we care about 2024-11-16T09:58:48,843 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=311 (bloomFilter=true), to=hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/7c7f87a6124bfe2f4049ee7803e0f98c/.tmp/C/9c7d3a5858cc46d2aeb57bf167db023f 2024-11-16T09:58:48,849 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/7c7f87a6124bfe2f4049ee7803e0f98c/.tmp/A/34e9400bce784c0fa95be143ab8eefe9 as hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/7c7f87a6124bfe2f4049ee7803e0f98c/A/34e9400bce784c0fa95be143ab8eefe9 2024-11-16T09:58:48,853 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/7c7f87a6124bfe2f4049ee7803e0f98c/A/34e9400bce784c0fa95be143ab8eefe9, entries=200, sequenceid=311, filesize=39.0 K 2024-11-16T09:58:48,854 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/7c7f87a6124bfe2f4049ee7803e0f98c/.tmp/B/724e91f5d6c845aeaf175fa51a5f0e39 as hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/7c7f87a6124bfe2f4049ee7803e0f98c/B/724e91f5d6c845aeaf175fa51a5f0e39 2024-11-16T09:58:48,858 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/7c7f87a6124bfe2f4049ee7803e0f98c/B/724e91f5d6c845aeaf175fa51a5f0e39, entries=150, sequenceid=311, filesize=12.0 K 2024-11-16T09:58:48,860 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/7c7f87a6124bfe2f4049ee7803e0f98c/.tmp/C/9c7d3a5858cc46d2aeb57bf167db023f as hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/7c7f87a6124bfe2f4049ee7803e0f98c/C/9c7d3a5858cc46d2aeb57bf167db023f 2024-11-16T09:58:48,864 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/7c7f87a6124bfe2f4049ee7803e0f98c/C/9c7d3a5858cc46d2aeb57bf167db023f, entries=150, sequenceid=311, filesize=12.0 K 2024-11-16T09:58:48,865 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~107.34 KB/109920, heapSize ~281.95 KB/288720, currentSize=93.93 KB/96180 for 7c7f87a6124bfe2f4049ee7803e0f98c in 1270ms, sequenceid=311, compaction requested=true 2024-11-16T09:58:48,865 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 7c7f87a6124bfe2f4049ee7803e0f98c: 2024-11-16T09:58:48,865 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 7c7f87a6124bfe2f4049ee7803e0f98c:A, priority=-2147483648, current under compaction store size is 1 2024-11-16T09:58:48,865 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-16T09:58:48,865 DEBUG [RS:0;037c58a19ddb:37267-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-16T09:58:48,865 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 7c7f87a6124bfe2f4049ee7803e0f98c:B, priority=-2147483648, current under compaction store size is 2 2024-11-16T09:58:48,865 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-16T09:58:48,865 DEBUG [RS:0;037c58a19ddb:37267-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-16T09:58:48,865 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 7c7f87a6124bfe2f4049ee7803e0f98c:C, priority=-2147483648, current under compaction store size is 3 2024-11-16T09:58:48,865 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-16T09:58:48,866 DEBUG [RS:0;037c58a19ddb:37267-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 103073 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-16T09:58:48,866 DEBUG [RS:0;037c58a19ddb:37267-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37517 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-16T09:58:48,866 DEBUG [RS:0;037c58a19ddb:37267-shortCompactions-0 {}] regionserver.HStore(1541): 7c7f87a6124bfe2f4049ee7803e0f98c/A is initiating minor compaction (all files) 2024-11-16T09:58:48,866 DEBUG [RS:0;037c58a19ddb:37267-longCompactions-0 {}] regionserver.HStore(1541): 7c7f87a6124bfe2f4049ee7803e0f98c/B is initiating minor compaction (all files) 2024-11-16T09:58:48,866 INFO [RS:0;037c58a19ddb:37267-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 7c7f87a6124bfe2f4049ee7803e0f98c/A in TestAcidGuarantees,,1731751108989.7c7f87a6124bfe2f4049ee7803e0f98c. 2024-11-16T09:58:48,866 INFO [RS:0;037c58a19ddb:37267-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 7c7f87a6124bfe2f4049ee7803e0f98c/B in TestAcidGuarantees,,1731751108989.7c7f87a6124bfe2f4049ee7803e0f98c. 2024-11-16T09:58:48,867 INFO [RS:0;037c58a19ddb:37267-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/7c7f87a6124bfe2f4049ee7803e0f98c/A/cf911fd937c34d99a23ab70841f56bfe, hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/7c7f87a6124bfe2f4049ee7803e0f98c/A/5ea7fb182fe64f0b9234c748529c1dec, hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/7c7f87a6124bfe2f4049ee7803e0f98c/A/34e9400bce784c0fa95be143ab8eefe9] into tmpdir=hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/7c7f87a6124bfe2f4049ee7803e0f98c/.tmp, totalSize=100.7 K 2024-11-16T09:58:48,867 INFO [RS:0;037c58a19ddb:37267-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/7c7f87a6124bfe2f4049ee7803e0f98c/B/82d565f1aaa9448aaf40a4bdcf5f4ddc, hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/7c7f87a6124bfe2f4049ee7803e0f98c/B/d5f199b181be423483c2bb9c67b0a823, hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/7c7f87a6124bfe2f4049ee7803e0f98c/B/724e91f5d6c845aeaf175fa51a5f0e39] into tmpdir=hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/7c7f87a6124bfe2f4049ee7803e0f98c/.tmp, totalSize=36.6 K 2024-11-16T09:58:48,867 INFO [RS:0;037c58a19ddb:37267-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731751108989.7c7f87a6124bfe2f4049ee7803e0f98c. 2024-11-16T09:58:48,867 DEBUG [RS:0;037c58a19ddb:37267-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731751108989.7c7f87a6124bfe2f4049ee7803e0f98c. files: [hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/7c7f87a6124bfe2f4049ee7803e0f98c/A/cf911fd937c34d99a23ab70841f56bfe, hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/7c7f87a6124bfe2f4049ee7803e0f98c/A/5ea7fb182fe64f0b9234c748529c1dec, hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/7c7f87a6124bfe2f4049ee7803e0f98c/A/34e9400bce784c0fa95be143ab8eefe9] 2024-11-16T09:58:48,867 DEBUG [RS:0;037c58a19ddb:37267-longCompactions-0 {}] compactions.Compactor(225): Compacting 82d565f1aaa9448aaf40a4bdcf5f4ddc, keycount=150, bloomtype=ROW, size=12.6 K, encoding=NONE, compression=NONE, seqNum=271, earliestPutTs=1731751123693 2024-11-16T09:58:48,867 DEBUG [RS:0;037c58a19ddb:37267-shortCompactions-0 {}] compactions.Compactor(225): Compacting cf911fd937c34d99a23ab70841f56bfe, keycount=150, bloomtype=ROW, size=31.1 K, encoding=NONE, compression=NONE, seqNum=271, earliestPutTs=1731751123693 2024-11-16T09:58:48,867 DEBUG [RS:0;037c58a19ddb:37267-longCompactions-0 {}] compactions.Compactor(225): Compacting d5f199b181be423483c2bb9c67b0a823, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=289, earliestPutTs=1731751124348 2024-11-16T09:58:48,867 DEBUG [RS:0;037c58a19ddb:37267-shortCompactions-0 {}] compactions.Compactor(225): Compacting 5ea7fb182fe64f0b9234c748529c1dec, keycount=150, bloomtype=ROW, size=30.5 K, encoding=NONE, compression=NONE, seqNum=289, earliestPutTs=1731751124348 2024-11-16T09:58:48,868 DEBUG [RS:0;037c58a19ddb:37267-longCompactions-0 {}] compactions.Compactor(225): Compacting 724e91f5d6c845aeaf175fa51a5f0e39, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=311, earliestPutTs=1731751126438 2024-11-16T09:58:48,868 DEBUG [RS:0;037c58a19ddb:37267-shortCompactions-0 {}] compactions.Compactor(225): Compacting 34e9400bce784c0fa95be143ab8eefe9, keycount=200, bloomtype=ROW, size=39.0 K, encoding=NONE, compression=NONE, seqNum=311, earliestPutTs=1731751126438 2024-11-16T09:58:48,875 INFO [RS:0;037c58a19ddb:37267-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=7c7f87a6124bfe2f4049ee7803e0f98c] 2024-11-16T09:58:48,877 INFO [RS:0;037c58a19ddb:37267-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 7c7f87a6124bfe2f4049ee7803e0f98c#B#compaction#192 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-16T09:58:48,877 DEBUG [RS:0;037c58a19ddb:37267-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/7c7f87a6124bfe2f4049ee7803e0f98c/.tmp/B/e21881a409cd409893e1543bb8f3f4d3 is 50, key is test_row_0/B:col10/1731751127593/Put/seqid=0 2024-11-16T09:58:48,879 DEBUG [RS:0;037c58a19ddb:37267-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e2024111687c9da05bde94cc7bcf2561feeb63153_7c7f87a6124bfe2f4049ee7803e0f98c store=[table=TestAcidGuarantees family=A region=7c7f87a6124bfe2f4049ee7803e0f98c] 2024-11-16T09:58:48,881 DEBUG [RS:0;037c58a19ddb:37267-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e2024111687c9da05bde94cc7bcf2561feeb63153_7c7f87a6124bfe2f4049ee7803e0f98c, store=[table=TestAcidGuarantees family=A region=7c7f87a6124bfe2f4049ee7803e0f98c] 2024-11-16T09:58:48,881 DEBUG [RS:0;037c58a19ddb:37267-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111687c9da05bde94cc7bcf2561feeb63153_7c7f87a6124bfe2f4049ee7803e0f98c because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=7c7f87a6124bfe2f4049ee7803e0f98c] 2024-11-16T09:58:48,883 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37591 is added to blk_1073742053_1229 (size=13017) 2024-11-16T09:58:48,889 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37591 is added to blk_1073742054_1230 (size=4469) 2024-11-16T09:58:48,890 INFO [RS:0;037c58a19ddb:37267-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 7c7f87a6124bfe2f4049ee7803e0f98c#A#compaction#193 average throughput is 1.63 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-16T09:58:48,891 DEBUG [RS:0;037c58a19ddb:37267-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/7c7f87a6124bfe2f4049ee7803e0f98c/.tmp/A/4ffc1030f8104b6ba06470b62b7108f6 is 175, key is test_row_0/A:col10/1731751127593/Put/seqid=0 2024-11-16T09:58:48,894 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37591 is added to blk_1073742055_1231 (size=31971) 2024-11-16T09:58:48,953 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37877 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=46 2024-11-16T09:58:48,953 INFO [RPCClient-NioEventLoopGroup-4-7 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-16T09:58:48,954 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37877 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-16T09:58:48,954 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37877 {}] procedure2.ProcedureExecutor(1139): Stored pid=48, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=48, table=TestAcidGuarantees 2024-11-16T09:58:48,955 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37877 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=48 2024-11-16T09:58:48,955 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=48, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=48, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-16T09:58:48,956 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=48, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=48, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-16T09:58:48,956 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=49, ppid=48, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-16T09:58:49,063 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37877 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=48 2024-11-16T09:58:49,109 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=37267 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=49 2024-11-16T09:58:49,109 DEBUG [RS_FLUSH_OPERATIONS-regionserver/037c58a19ddb:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731751108989.7c7f87a6124bfe2f4049ee7803e0f98c. 2024-11-16T09:58:49,109 INFO [RS_FLUSH_OPERATIONS-regionserver/037c58a19ddb:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HRegion(2902): Flushing 7c7f87a6124bfe2f4049ee7803e0f98c 3/3 column families, dataSize=93.93 KB heapSize=246.84 KB 2024-11-16T09:58:49,117 DEBUG [RS_FLUSH_OPERATIONS-regionserver/037c58a19ddb:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411167774bab456c84fd3bb8f867e522b2ee6_7c7f87a6124bfe2f4049ee7803e0f98c is 50, key is test_row_0/A:col10/1731751127657/Put/seqid=0 2024-11-16T09:58:49,121 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37591 is added to blk_1073742056_1232 (size=12454) 2024-11-16T09:58:49,273 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37877 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=48 2024-11-16T09:58:49,289 DEBUG [RS:0;037c58a19ddb:37267-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/7c7f87a6124bfe2f4049ee7803e0f98c/.tmp/B/e21881a409cd409893e1543bb8f3f4d3 as hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/7c7f87a6124bfe2f4049ee7803e0f98c/B/e21881a409cd409893e1543bb8f3f4d3 2024-11-16T09:58:49,295 INFO [RS:0;037c58a19ddb:37267-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 7c7f87a6124bfe2f4049ee7803e0f98c/B of 7c7f87a6124bfe2f4049ee7803e0f98c into e21881a409cd409893e1543bb8f3f4d3(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-16T09:58:49,295 DEBUG [RS:0;037c58a19ddb:37267-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 7c7f87a6124bfe2f4049ee7803e0f98c: 2024-11-16T09:58:49,295 INFO [RS:0;037c58a19ddb:37267-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731751108989.7c7f87a6124bfe2f4049ee7803e0f98c., storeName=7c7f87a6124bfe2f4049ee7803e0f98c/B, priority=13, startTime=1731751128865; duration=0sec 2024-11-16T09:58:49,295 DEBUG [RS:0;037c58a19ddb:37267-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-16T09:58:49,295 DEBUG [RS:0;037c58a19ddb:37267-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 7c7f87a6124bfe2f4049ee7803e0f98c:B 2024-11-16T09:58:49,295 DEBUG [RS:0;037c58a19ddb:37267-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-16T09:58:49,297 DEBUG [RS:0;037c58a19ddb:37267-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37517 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-16T09:58:49,297 DEBUG [RS:0;037c58a19ddb:37267-longCompactions-0 {}] regionserver.HStore(1541): 7c7f87a6124bfe2f4049ee7803e0f98c/C is initiating minor compaction (all files) 2024-11-16T09:58:49,297 INFO [RS:0;037c58a19ddb:37267-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 7c7f87a6124bfe2f4049ee7803e0f98c/C in TestAcidGuarantees,,1731751108989.7c7f87a6124bfe2f4049ee7803e0f98c. 2024-11-16T09:58:49,297 INFO [RS:0;037c58a19ddb:37267-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/7c7f87a6124bfe2f4049ee7803e0f98c/C/c9f68351e8204e01acd0f22f12b4cc64, hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/7c7f87a6124bfe2f4049ee7803e0f98c/C/c8cbef2eeb3749d4b5fe5998814b7135, hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/7c7f87a6124bfe2f4049ee7803e0f98c/C/9c7d3a5858cc46d2aeb57bf167db023f] into tmpdir=hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/7c7f87a6124bfe2f4049ee7803e0f98c/.tmp, totalSize=36.6 K 2024-11-16T09:58:49,298 DEBUG [RS:0;037c58a19ddb:37267-longCompactions-0 {}] compactions.Compactor(225): Compacting c9f68351e8204e01acd0f22f12b4cc64, keycount=150, bloomtype=ROW, size=12.6 K, encoding=NONE, compression=NONE, seqNum=271, earliestPutTs=1731751123693 2024-11-16T09:58:49,298 DEBUG [RS:0;037c58a19ddb:37267-longCompactions-0 {}] compactions.Compactor(225): Compacting c8cbef2eeb3749d4b5fe5998814b7135, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=289, earliestPutTs=1731751124348 2024-11-16T09:58:49,299 DEBUG [RS:0;037c58a19ddb:37267-longCompactions-0 {}] compactions.Compactor(225): Compacting 9c7d3a5858cc46d2aeb57bf167db023f, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=311, earliestPutTs=1731751126438 2024-11-16T09:58:49,300 DEBUG [RS:0;037c58a19ddb:37267-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/7c7f87a6124bfe2f4049ee7803e0f98c/.tmp/A/4ffc1030f8104b6ba06470b62b7108f6 as hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/7c7f87a6124bfe2f4049ee7803e0f98c/A/4ffc1030f8104b6ba06470b62b7108f6 2024-11-16T09:58:49,306 INFO [RS:0;037c58a19ddb:37267-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 7c7f87a6124bfe2f4049ee7803e0f98c/A of 7c7f87a6124bfe2f4049ee7803e0f98c into 4ffc1030f8104b6ba06470b62b7108f6(size=31.2 K), total size for store is 31.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-16T09:58:49,306 DEBUG [RS:0;037c58a19ddb:37267-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 7c7f87a6124bfe2f4049ee7803e0f98c: 2024-11-16T09:58:49,306 INFO [RS:0;037c58a19ddb:37267-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731751108989.7c7f87a6124bfe2f4049ee7803e0f98c., storeName=7c7f87a6124bfe2f4049ee7803e0f98c/A, priority=13, startTime=1731751128865; duration=0sec 2024-11-16T09:58:49,306 DEBUG [RS:0;037c58a19ddb:37267-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-16T09:58:49,307 DEBUG [RS:0;037c58a19ddb:37267-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 7c7f87a6124bfe2f4049ee7803e0f98c:A 2024-11-16T09:58:49,308 INFO [RS:0;037c58a19ddb:37267-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 7c7f87a6124bfe2f4049ee7803e0f98c#C#compaction#195 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-16T09:58:49,309 DEBUG [RS:0;037c58a19ddb:37267-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/7c7f87a6124bfe2f4049ee7803e0f98c/.tmp/C/713ae3ec75aa419e8acd7ec473839cc8 is 50, key is test_row_0/C:col10/1731751127593/Put/seqid=0 2024-11-16T09:58:49,317 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37591 is added to blk_1073742057_1233 (size=13017) 2024-11-16T09:58:49,522 DEBUG [RS_FLUSH_OPERATIONS-regionserver/037c58a19ddb:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T09:58:49,527 INFO [RS_FLUSH_OPERATIONS-regionserver/037c58a19ddb:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411167774bab456c84fd3bb8f867e522b2ee6_7c7f87a6124bfe2f4049ee7803e0f98c to hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411167774bab456c84fd3bb8f867e522b2ee6_7c7f87a6124bfe2f4049ee7803e0f98c 2024-11-16T09:58:49,528 DEBUG [RS_FLUSH_OPERATIONS-regionserver/037c58a19ddb:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/7c7f87a6124bfe2f4049ee7803e0f98c/.tmp/A/76d20ead6b9f46678cadd75d8a84ae44, store: [table=TestAcidGuarantees family=A region=7c7f87a6124bfe2f4049ee7803e0f98c] 2024-11-16T09:58:49,529 DEBUG [RS_FLUSH_OPERATIONS-regionserver/037c58a19ddb:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/7c7f87a6124bfe2f4049ee7803e0f98c/.tmp/A/76d20ead6b9f46678cadd75d8a84ae44 is 175, key is test_row_0/A:col10/1731751127657/Put/seqid=0 2024-11-16T09:58:49,534 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37591 is added to blk_1073742058_1234 (size=31255) 2024-11-16T09:58:49,583 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37877 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=48 2024-11-16T09:58:49,722 DEBUG [RS:0;037c58a19ddb:37267-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/7c7f87a6124bfe2f4049ee7803e0f98c/.tmp/C/713ae3ec75aa419e8acd7ec473839cc8 as hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/7c7f87a6124bfe2f4049ee7803e0f98c/C/713ae3ec75aa419e8acd7ec473839cc8 2024-11-16T09:58:49,727 INFO [RS:0;037c58a19ddb:37267-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 7c7f87a6124bfe2f4049ee7803e0f98c/C of 7c7f87a6124bfe2f4049ee7803e0f98c into 713ae3ec75aa419e8acd7ec473839cc8(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-16T09:58:49,727 DEBUG [RS:0;037c58a19ddb:37267-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 7c7f87a6124bfe2f4049ee7803e0f98c: 2024-11-16T09:58:49,727 INFO [RS:0;037c58a19ddb:37267-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731751108989.7c7f87a6124bfe2f4049ee7803e0f98c., storeName=7c7f87a6124bfe2f4049ee7803e0f98c/C, priority=13, startTime=1731751128865; duration=0sec 2024-11-16T09:58:49,727 DEBUG [RS:0;037c58a19ddb:37267-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-16T09:58:49,727 DEBUG [RS:0;037c58a19ddb:37267-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 7c7f87a6124bfe2f4049ee7803e0f98c:C 2024-11-16T09:58:49,825 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] regionserver.HRegion(8855): Flush requested on 7c7f87a6124bfe2f4049ee7803e0f98c 2024-11-16T09:58:49,825 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731751108989.7c7f87a6124bfe2f4049ee7803e0f98c. as already flushing 2024-11-16T09:58:49,881 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c7f87a6124bfe2f4049ee7803e0f98c, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T09:58:49,881 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] ipc.CallRunner(138): callId: 99 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52428 deadline: 1731751189877, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c7f87a6124bfe2f4049ee7803e0f98c, server=037c58a19ddb,37267,1731751077844 2024-11-16T09:58:49,881 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731751108989.7c7f87a6124bfe2f4049ee7803e0f98c., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 , the old value is region=TestAcidGuarantees,,1731751108989.7c7f87a6124bfe2f4049ee7803e0f98c., hostname=037c58a19ddb,37267,1731751077844, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c7f87a6124bfe2f4049ee7803e0f98c, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:58:49,881 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731751108989.7c7f87a6124bfe2f4049ee7803e0f98c., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c7f87a6124bfe2f4049ee7803e0f98c, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:58:49,881 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731751108989.7c7f87a6124bfe2f4049ee7803e0f98c., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 because the exception is null or not the one we care about 2024-11-16T09:58:49,935 INFO [RS_FLUSH_OPERATIONS-regionserver/037c58a19ddb:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=328, memsize=31.3 K, hasBloomFilter=true, into tmp file hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/7c7f87a6124bfe2f4049ee7803e0f98c/.tmp/A/76d20ead6b9f46678cadd75d8a84ae44 2024-11-16T09:58:49,943 DEBUG [RS_FLUSH_OPERATIONS-regionserver/037c58a19ddb:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/7c7f87a6124bfe2f4049ee7803e0f98c/.tmp/B/0f9b87aa526f4d468458a3048268b749 is 50, key is test_row_0/B:col10/1731751127657/Put/seqid=0 2024-11-16T09:58:49,948 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37591 is added to blk_1073742059_1235 (size=12301) 2024-11-16T09:58:49,984 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c7f87a6124bfe2f4049ee7803e0f98c, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T09:58:49,984 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] ipc.CallRunner(138): callId: 100 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52428 deadline: 1731751189983, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c7f87a6124bfe2f4049ee7803e0f98c, server=037c58a19ddb,37267,1731751077844 2024-11-16T09:58:49,984 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731751108989.7c7f87a6124bfe2f4049ee7803e0f98c., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 , the old value is region=TestAcidGuarantees,,1731751108989.7c7f87a6124bfe2f4049ee7803e0f98c., hostname=037c58a19ddb,37267,1731751077844, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c7f87a6124bfe2f4049ee7803e0f98c, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:58:49,984 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731751108989.7c7f87a6124bfe2f4049ee7803e0f98c., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c7f87a6124bfe2f4049ee7803e0f98c, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:58:49,985 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731751108989.7c7f87a6124bfe2f4049ee7803e0f98c., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 because the exception is null or not the one we care about 2024-11-16T09:58:50,093 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37877 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=48 2024-11-16T09:58:50,193 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c7f87a6124bfe2f4049ee7803e0f98c, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T09:58:50,194 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] ipc.CallRunner(138): callId: 101 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52428 deadline: 1731751190192, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c7f87a6124bfe2f4049ee7803e0f98c, server=037c58a19ddb,37267,1731751077844 2024-11-16T09:58:50,194 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731751108989.7c7f87a6124bfe2f4049ee7803e0f98c., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 , the old value is region=TestAcidGuarantees,,1731751108989.7c7f87a6124bfe2f4049ee7803e0f98c., hostname=037c58a19ddb,37267,1731751077844, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c7f87a6124bfe2f4049ee7803e0f98c, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:58:50,194 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731751108989.7c7f87a6124bfe2f4049ee7803e0f98c., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c7f87a6124bfe2f4049ee7803e0f98c, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:58:50,194 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731751108989.7c7f87a6124bfe2f4049ee7803e0f98c., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 because the exception is null or not the one we care about 2024-11-16T09:58:50,348 INFO [RS_FLUSH_OPERATIONS-regionserver/037c58a19ddb:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=328 (bloomFilter=true), to=hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/7c7f87a6124bfe2f4049ee7803e0f98c/.tmp/B/0f9b87aa526f4d468458a3048268b749 2024-11-16T09:58:50,356 DEBUG [RS_FLUSH_OPERATIONS-regionserver/037c58a19ddb:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/7c7f87a6124bfe2f4049ee7803e0f98c/.tmp/C/9b4c5b55ab9a47fb97b7f974d882af88 is 50, key is test_row_0/C:col10/1731751127657/Put/seqid=0 2024-11-16T09:58:50,360 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37591 is added to blk_1073742060_1236 (size=12301) 2024-11-16T09:58:50,505 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c7f87a6124bfe2f4049ee7803e0f98c, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T09:58:50,505 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] ipc.CallRunner(138): callId: 102 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52428 deadline: 1731751190503, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c7f87a6124bfe2f4049ee7803e0f98c, server=037c58a19ddb,37267,1731751077844 2024-11-16T09:58:50,506 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731751108989.7c7f87a6124bfe2f4049ee7803e0f98c., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 , the old value is region=TestAcidGuarantees,,1731751108989.7c7f87a6124bfe2f4049ee7803e0f98c., hostname=037c58a19ddb,37267,1731751077844, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c7f87a6124bfe2f4049ee7803e0f98c, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:58:50,506 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731751108989.7c7f87a6124bfe2f4049ee7803e0f98c., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c7f87a6124bfe2f4049ee7803e0f98c, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:58:50,506 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731751108989.7c7f87a6124bfe2f4049ee7803e0f98c., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 because the exception is null or not the one we care about 2024-11-16T09:58:50,762 INFO [RS_FLUSH_OPERATIONS-regionserver/037c58a19ddb:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=328 (bloomFilter=true), to=hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/7c7f87a6124bfe2f4049ee7803e0f98c/.tmp/C/9b4c5b55ab9a47fb97b7f974d882af88 2024-11-16T09:58:50,767 DEBUG [RS_FLUSH_OPERATIONS-regionserver/037c58a19ddb:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/7c7f87a6124bfe2f4049ee7803e0f98c/.tmp/A/76d20ead6b9f46678cadd75d8a84ae44 as hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/7c7f87a6124bfe2f4049ee7803e0f98c/A/76d20ead6b9f46678cadd75d8a84ae44 2024-11-16T09:58:50,771 INFO [RS_FLUSH_OPERATIONS-regionserver/037c58a19ddb:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/7c7f87a6124bfe2f4049ee7803e0f98c/A/76d20ead6b9f46678cadd75d8a84ae44, entries=150, sequenceid=328, filesize=30.5 K 2024-11-16T09:58:50,773 DEBUG [RS_FLUSH_OPERATIONS-regionserver/037c58a19ddb:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/7c7f87a6124bfe2f4049ee7803e0f98c/.tmp/B/0f9b87aa526f4d468458a3048268b749 as hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/7c7f87a6124bfe2f4049ee7803e0f98c/B/0f9b87aa526f4d468458a3048268b749 2024-11-16T09:58:50,777 INFO [RS_FLUSH_OPERATIONS-regionserver/037c58a19ddb:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/7c7f87a6124bfe2f4049ee7803e0f98c/B/0f9b87aa526f4d468458a3048268b749, entries=150, sequenceid=328, filesize=12.0 K 2024-11-16T09:58:50,778 DEBUG [RS_FLUSH_OPERATIONS-regionserver/037c58a19ddb:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/7c7f87a6124bfe2f4049ee7803e0f98c/.tmp/C/9b4c5b55ab9a47fb97b7f974d882af88 as hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/7c7f87a6124bfe2f4049ee7803e0f98c/C/9b4c5b55ab9a47fb97b7f974d882af88 2024-11-16T09:58:50,782 INFO [RS_FLUSH_OPERATIONS-regionserver/037c58a19ddb:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/7c7f87a6124bfe2f4049ee7803e0f98c/C/9b4c5b55ab9a47fb97b7f974d882af88, entries=150, sequenceid=328, filesize=12.0 K 2024-11-16T09:58:50,783 INFO [RS_FLUSH_OPERATIONS-regionserver/037c58a19ddb:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HRegion(3140): Finished flush of dataSize ~93.93 KB/96180, heapSize ~246.80 KB/252720, currentSize=107.34 KB/109920 for 7c7f87a6124bfe2f4049ee7803e0f98c in 1673ms, sequenceid=328, compaction requested=false 2024-11-16T09:58:50,783 DEBUG [RS_FLUSH_OPERATIONS-regionserver/037c58a19ddb:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HRegion(2603): Flush status journal for 7c7f87a6124bfe2f4049ee7803e0f98c: 2024-11-16T09:58:50,783 DEBUG [RS_FLUSH_OPERATIONS-regionserver/037c58a19ddb:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731751108989.7c7f87a6124bfe2f4049ee7803e0f98c. 2024-11-16T09:58:50,783 DEBUG [RS_FLUSH_OPERATIONS-regionserver/037c58a19ddb:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=49 2024-11-16T09:58:50,783 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37877 {}] master.HMaster(4169): Remote procedure done, pid=49 2024-11-16T09:58:50,786 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=49, resume processing ppid=48 2024-11-16T09:58:50,786 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=49, ppid=48, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.8280 sec 2024-11-16T09:58:50,788 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=48, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=48, table=TestAcidGuarantees in 1.8330 sec 2024-11-16T09:58:51,015 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] regionserver.HRegion(8855): Flush requested on 7c7f87a6124bfe2f4049ee7803e0f98c 2024-11-16T09:58:51,016 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-16T09:58:51,016 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 7c7f87a6124bfe2f4049ee7803e0f98c 3/3 column families, dataSize=114.05 KB heapSize=299.58 KB 2024-11-16T09:58:51,024 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241116037412d84b294a3bacdddbe69c227702_7c7f87a6124bfe2f4049ee7803e0f98c is 50, key is test_row_0/A:col10/1731751129871/Put/seqid=0 2024-11-16T09:58:51,030 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37591 is added to blk_1073742061_1237 (size=14994) 2024-11-16T09:58:51,076 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c7f87a6124bfe2f4049ee7803e0f98c, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T09:58:51,077 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] ipc.CallRunner(138): callId: 117 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52428 deadline: 1731751191073, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c7f87a6124bfe2f4049ee7803e0f98c, server=037c58a19ddb,37267,1731751077844 2024-11-16T09:58:51,077 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731751108989.7c7f87a6124bfe2f4049ee7803e0f98c., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 , the old value is region=TestAcidGuarantees,,1731751108989.7c7f87a6124bfe2f4049ee7803e0f98c., hostname=037c58a19ddb,37267,1731751077844, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c7f87a6124bfe2f4049ee7803e0f98c, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:58:51,077 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731751108989.7c7f87a6124bfe2f4049ee7803e0f98c., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c7f87a6124bfe2f4049ee7803e0f98c, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:58:51,077 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731751108989.7c7f87a6124bfe2f4049ee7803e0f98c., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 because the exception is null or not the one we care about 2024-11-16T09:58:51,103 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37877 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=48 2024-11-16T09:58:51,103 INFO [RPCClient-NioEventLoopGroup-4-7 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-16T09:58:51,104 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37877 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-16T09:58:51,105 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37877 {}] procedure2.ProcedureExecutor(1139): Stored pid=50, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=50, table=TestAcidGuarantees 2024-11-16T09:58:51,105 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37877 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=50 2024-11-16T09:58:51,106 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=50, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=50, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-16T09:58:51,106 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=50, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=50, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-16T09:58:51,106 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=51, ppid=50, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-16T09:58:51,184 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c7f87a6124bfe2f4049ee7803e0f98c, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T09:58:51,185 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] ipc.CallRunner(138): callId: 118 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52428 deadline: 1731751191183, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c7f87a6124bfe2f4049ee7803e0f98c, server=037c58a19ddb,37267,1731751077844 2024-11-16T09:58:51,185 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731751108989.7c7f87a6124bfe2f4049ee7803e0f98c., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 , the old value is region=TestAcidGuarantees,,1731751108989.7c7f87a6124bfe2f4049ee7803e0f98c., hostname=037c58a19ddb,37267,1731751077844, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c7f87a6124bfe2f4049ee7803e0f98c, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:58:51,185 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731751108989.7c7f87a6124bfe2f4049ee7803e0f98c., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c7f87a6124bfe2f4049ee7803e0f98c, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:58:51,185 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731751108989.7c7f87a6124bfe2f4049ee7803e0f98c., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 because the exception is null or not the one we care about 2024-11-16T09:58:51,213 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37877 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=50 2024-11-16T09:58:51,257 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=37267 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=51 2024-11-16T09:58:51,258 DEBUG [RS_FLUSH_OPERATIONS-regionserver/037c58a19ddb:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731751108989.7c7f87a6124bfe2f4049ee7803e0f98c. 2024-11-16T09:58:51,258 DEBUG [RS_FLUSH_OPERATIONS-regionserver/037c58a19ddb:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731751108989.7c7f87a6124bfe2f4049ee7803e0f98c. as already flushing 2024-11-16T09:58:51,258 DEBUG [RS_FLUSH_OPERATIONS-regionserver/037c58a19ddb:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731751108989.7c7f87a6124bfe2f4049ee7803e0f98c. 2024-11-16T09:58:51,258 ERROR [RS_FLUSH_OPERATIONS-regionserver/037c58a19ddb:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] handler.RSProcedureHandler(58): pid=51 java.io.IOException: Unable to complete flush {ENCODED => 7c7f87a6124bfe2f4049ee7803e0f98c, NAME => 'TestAcidGuarantees,,1731751108989.7c7f87a6124bfe2f4049ee7803e0f98c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T09:58:51,258 DEBUG [RS_FLUSH_OPERATIONS-regionserver/037c58a19ddb:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=51 java.io.IOException: Unable to complete flush {ENCODED => 7c7f87a6124bfe2f4049ee7803e0f98c, NAME => 'TestAcidGuarantees,,1731751108989.7c7f87a6124bfe2f4049ee7803e0f98c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T09:58:51,259 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37877 {}] master.HMaster(4177): Remote procedure failed, pid=51 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 7c7f87a6124bfe2f4049ee7803e0f98c, NAME => 'TestAcidGuarantees,,1731751108989.7c7f87a6124bfe2f4049ee7803e0f98c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 7c7f87a6124bfe2f4049ee7803e0f98c, NAME => 'TestAcidGuarantees,,1731751108989.7c7f87a6124bfe2f4049ee7803e0f98c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T09:58:51,398 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c7f87a6124bfe2f4049ee7803e0f98c, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T09:58:51,398 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] ipc.CallRunner(138): callId: 119 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52428 deadline: 1731751191393, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c7f87a6124bfe2f4049ee7803e0f98c, server=037c58a19ddb,37267,1731751077844 2024-11-16T09:58:51,399 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731751108989.7c7f87a6124bfe2f4049ee7803e0f98c., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 , the old value is region=TestAcidGuarantees,,1731751108989.7c7f87a6124bfe2f4049ee7803e0f98c., hostname=037c58a19ddb,37267,1731751077844, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c7f87a6124bfe2f4049ee7803e0f98c, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:58:51,399 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731751108989.7c7f87a6124bfe2f4049ee7803e0f98c., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c7f87a6124bfe2f4049ee7803e0f98c, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:58:51,399 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731751108989.7c7f87a6124bfe2f4049ee7803e0f98c., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 because the exception is null or not the one we care about 2024-11-16T09:58:51,410 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=37267 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=51 2024-11-16T09:58:51,411 DEBUG [RS_FLUSH_OPERATIONS-regionserver/037c58a19ddb:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731751108989.7c7f87a6124bfe2f4049ee7803e0f98c. 2024-11-16T09:58:51,411 DEBUG [RS_FLUSH_OPERATIONS-regionserver/037c58a19ddb:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731751108989.7c7f87a6124bfe2f4049ee7803e0f98c. as already flushing 2024-11-16T09:58:51,411 DEBUG [RS_FLUSH_OPERATIONS-regionserver/037c58a19ddb:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731751108989.7c7f87a6124bfe2f4049ee7803e0f98c. 2024-11-16T09:58:51,411 ERROR [RS_FLUSH_OPERATIONS-regionserver/037c58a19ddb:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] handler.RSProcedureHandler(58): pid=51 java.io.IOException: Unable to complete flush {ENCODED => 7c7f87a6124bfe2f4049ee7803e0f98c, NAME => 'TestAcidGuarantees,,1731751108989.7c7f87a6124bfe2f4049ee7803e0f98c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T09:58:51,411 DEBUG [RS_FLUSH_OPERATIONS-regionserver/037c58a19ddb:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=51 java.io.IOException: Unable to complete flush {ENCODED => 7c7f87a6124bfe2f4049ee7803e0f98c, NAME => 'TestAcidGuarantees,,1731751108989.7c7f87a6124bfe2f4049ee7803e0f98c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T09:58:51,412 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37877 {}] master.HMaster(4177): Remote procedure failed, pid=51 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 7c7f87a6124bfe2f4049ee7803e0f98c, NAME => 'TestAcidGuarantees,,1731751108989.7c7f87a6124bfe2f4049ee7803e0f98c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 7c7f87a6124bfe2f4049ee7803e0f98c, NAME => 'TestAcidGuarantees,,1731751108989.7c7f87a6124bfe2f4049ee7803e0f98c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T09:58:51,423 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37877 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=50 2024-11-16T09:58:51,431 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T09:58:51,435 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241116037412d84b294a3bacdddbe69c227702_7c7f87a6124bfe2f4049ee7803e0f98c to hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241116037412d84b294a3bacdddbe69c227702_7c7f87a6124bfe2f4049ee7803e0f98c 2024-11-16T09:58:51,437 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/7c7f87a6124bfe2f4049ee7803e0f98c/.tmp/A/d3490b0bbc89425ca64d2102a813fa1f, store: [table=TestAcidGuarantees family=A region=7c7f87a6124bfe2f4049ee7803e0f98c] 2024-11-16T09:58:51,437 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/7c7f87a6124bfe2f4049ee7803e0f98c/.tmp/A/d3490b0bbc89425ca64d2102a813fa1f is 175, key is test_row_0/A:col10/1731751129871/Put/seqid=0 2024-11-16T09:58:51,445 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37591 is added to blk_1073742062_1238 (size=39949) 2024-11-16T09:58:51,446 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=351, memsize=38.0 K, hasBloomFilter=true, into tmp file hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/7c7f87a6124bfe2f4049ee7803e0f98c/.tmp/A/d3490b0bbc89425ca64d2102a813fa1f 2024-11-16T09:58:51,455 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/7c7f87a6124bfe2f4049ee7803e0f98c/.tmp/B/96fd5ab09a8e47cba98b0b1f2bcda7a3 is 50, key is test_row_0/B:col10/1731751129871/Put/seqid=0 2024-11-16T09:58:51,462 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37591 is added to blk_1073742063_1239 (size=12301) 2024-11-16T09:58:51,463 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=351 (bloomFilter=true), to=hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/7c7f87a6124bfe2f4049ee7803e0f98c/.tmp/B/96fd5ab09a8e47cba98b0b1f2bcda7a3 2024-11-16T09:58:51,471 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/7c7f87a6124bfe2f4049ee7803e0f98c/.tmp/C/745a570470de439d839401a55fbe1de1 is 50, key is test_row_0/C:col10/1731751129871/Put/seqid=0 2024-11-16T09:58:51,484 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37591 is added to blk_1073742064_1240 (size=12301) 2024-11-16T09:58:51,564 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=37267 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=51 2024-11-16T09:58:51,564 DEBUG [RS_FLUSH_OPERATIONS-regionserver/037c58a19ddb:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731751108989.7c7f87a6124bfe2f4049ee7803e0f98c. 2024-11-16T09:58:51,564 DEBUG [RS_FLUSH_OPERATIONS-regionserver/037c58a19ddb:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731751108989.7c7f87a6124bfe2f4049ee7803e0f98c. as already flushing 2024-11-16T09:58:51,564 DEBUG [RS_FLUSH_OPERATIONS-regionserver/037c58a19ddb:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731751108989.7c7f87a6124bfe2f4049ee7803e0f98c. 2024-11-16T09:58:51,564 ERROR [RS_FLUSH_OPERATIONS-regionserver/037c58a19ddb:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] handler.RSProcedureHandler(58): pid=51 java.io.IOException: Unable to complete flush {ENCODED => 7c7f87a6124bfe2f4049ee7803e0f98c, NAME => 'TestAcidGuarantees,,1731751108989.7c7f87a6124bfe2f4049ee7803e0f98c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T09:58:51,564 DEBUG [RS_FLUSH_OPERATIONS-regionserver/037c58a19ddb:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=51 java.io.IOException: Unable to complete flush {ENCODED => 7c7f87a6124bfe2f4049ee7803e0f98c, NAME => 'TestAcidGuarantees,,1731751108989.7c7f87a6124bfe2f4049ee7803e0f98c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T09:58:51,565 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37877 {}] master.HMaster(4177): Remote procedure failed, pid=51 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 7c7f87a6124bfe2f4049ee7803e0f98c, NAME => 'TestAcidGuarantees,,1731751108989.7c7f87a6124bfe2f4049ee7803e0f98c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 7c7f87a6124bfe2f4049ee7803e0f98c, NAME => 'TestAcidGuarantees,,1731751108989.7c7f87a6124bfe2f4049ee7803e0f98c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T09:58:51,706 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c7f87a6124bfe2f4049ee7803e0f98c, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T09:58:51,706 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] ipc.CallRunner(138): callId: 120 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52428 deadline: 1731751191703, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c7f87a6124bfe2f4049ee7803e0f98c, server=037c58a19ddb,37267,1731751077844 2024-11-16T09:58:51,706 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731751108989.7c7f87a6124bfe2f4049ee7803e0f98c., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 , the old value is region=TestAcidGuarantees,,1731751108989.7c7f87a6124bfe2f4049ee7803e0f98c., hostname=037c58a19ddb,37267,1731751077844, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c7f87a6124bfe2f4049ee7803e0f98c, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:58:51,706 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731751108989.7c7f87a6124bfe2f4049ee7803e0f98c., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c7f87a6124bfe2f4049ee7803e0f98c, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:58:51,706 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731751108989.7c7f87a6124bfe2f4049ee7803e0f98c., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 because the exception is null or not the one we care about 2024-11-16T09:58:51,717 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=37267 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=51 2024-11-16T09:58:51,717 DEBUG [RS_FLUSH_OPERATIONS-regionserver/037c58a19ddb:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731751108989.7c7f87a6124bfe2f4049ee7803e0f98c. 2024-11-16T09:58:51,717 DEBUG [RS_FLUSH_OPERATIONS-regionserver/037c58a19ddb:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731751108989.7c7f87a6124bfe2f4049ee7803e0f98c. as already flushing 2024-11-16T09:58:51,717 DEBUG [RS_FLUSH_OPERATIONS-regionserver/037c58a19ddb:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731751108989.7c7f87a6124bfe2f4049ee7803e0f98c. 2024-11-16T09:58:51,717 ERROR [RS_FLUSH_OPERATIONS-regionserver/037c58a19ddb:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] handler.RSProcedureHandler(58): pid=51 java.io.IOException: Unable to complete flush {ENCODED => 7c7f87a6124bfe2f4049ee7803e0f98c, NAME => 'TestAcidGuarantees,,1731751108989.7c7f87a6124bfe2f4049ee7803e0f98c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T09:58:51,718 DEBUG [RS_FLUSH_OPERATIONS-regionserver/037c58a19ddb:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=51 java.io.IOException: Unable to complete flush {ENCODED => 7c7f87a6124bfe2f4049ee7803e0f98c, NAME => 'TestAcidGuarantees,,1731751108989.7c7f87a6124bfe2f4049ee7803e0f98c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T09:58:51,718 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37877 {}] master.HMaster(4177): Remote procedure failed, pid=51 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 7c7f87a6124bfe2f4049ee7803e0f98c, NAME => 'TestAcidGuarantees,,1731751108989.7c7f87a6124bfe2f4049ee7803e0f98c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 7c7f87a6124bfe2f4049ee7803e0f98c, NAME => 'TestAcidGuarantees,,1731751108989.7c7f87a6124bfe2f4049ee7803e0f98c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T09:58:51,733 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37877 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=50 2024-11-16T09:58:51,870 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=37267 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=51 2024-11-16T09:58:51,870 DEBUG [RS_FLUSH_OPERATIONS-regionserver/037c58a19ddb:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731751108989.7c7f87a6124bfe2f4049ee7803e0f98c. 2024-11-16T09:58:51,871 DEBUG [RS_FLUSH_OPERATIONS-regionserver/037c58a19ddb:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731751108989.7c7f87a6124bfe2f4049ee7803e0f98c. as already flushing 2024-11-16T09:58:51,871 DEBUG [RS_FLUSH_OPERATIONS-regionserver/037c58a19ddb:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731751108989.7c7f87a6124bfe2f4049ee7803e0f98c. 2024-11-16T09:58:51,871 ERROR [RS_FLUSH_OPERATIONS-regionserver/037c58a19ddb:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] handler.RSProcedureHandler(58): pid=51 java.io.IOException: Unable to complete flush {ENCODED => 7c7f87a6124bfe2f4049ee7803e0f98c, NAME => 'TestAcidGuarantees,,1731751108989.7c7f87a6124bfe2f4049ee7803e0f98c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T09:58:51,871 DEBUG [RS_FLUSH_OPERATIONS-regionserver/037c58a19ddb:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=51 java.io.IOException: Unable to complete flush {ENCODED => 7c7f87a6124bfe2f4049ee7803e0f98c, NAME => 'TestAcidGuarantees,,1731751108989.7c7f87a6124bfe2f4049ee7803e0f98c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T09:58:51,872 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37877 {}] master.HMaster(4177): Remote procedure failed, pid=51 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 7c7f87a6124bfe2f4049ee7803e0f98c, NAME => 'TestAcidGuarantees,,1731751108989.7c7f87a6124bfe2f4049ee7803e0f98c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 7c7f87a6124bfe2f4049ee7803e0f98c, NAME => 'TestAcidGuarantees,,1731751108989.7c7f87a6124bfe2f4049ee7803e0f98c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T09:58:51,885 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=351 (bloomFilter=true), to=hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/7c7f87a6124bfe2f4049ee7803e0f98c/.tmp/C/745a570470de439d839401a55fbe1de1 2024-11-16T09:58:51,890 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/7c7f87a6124bfe2f4049ee7803e0f98c/.tmp/A/d3490b0bbc89425ca64d2102a813fa1f as hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/7c7f87a6124bfe2f4049ee7803e0f98c/A/d3490b0bbc89425ca64d2102a813fa1f 2024-11-16T09:58:51,894 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/7c7f87a6124bfe2f4049ee7803e0f98c/A/d3490b0bbc89425ca64d2102a813fa1f, entries=200, sequenceid=351, filesize=39.0 K 2024-11-16T09:58:51,895 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/7c7f87a6124bfe2f4049ee7803e0f98c/.tmp/B/96fd5ab09a8e47cba98b0b1f2bcda7a3 as hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/7c7f87a6124bfe2f4049ee7803e0f98c/B/96fd5ab09a8e47cba98b0b1f2bcda7a3 2024-11-16T09:58:51,900 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/7c7f87a6124bfe2f4049ee7803e0f98c/B/96fd5ab09a8e47cba98b0b1f2bcda7a3, entries=150, sequenceid=351, filesize=12.0 K 2024-11-16T09:58:51,901 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/7c7f87a6124bfe2f4049ee7803e0f98c/.tmp/C/745a570470de439d839401a55fbe1de1 as hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/7c7f87a6124bfe2f4049ee7803e0f98c/C/745a570470de439d839401a55fbe1de1 2024-11-16T09:58:51,905 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/7c7f87a6124bfe2f4049ee7803e0f98c/C/745a570470de439d839401a55fbe1de1, entries=150, sequenceid=351, filesize=12.0 K 2024-11-16T09:58:51,906 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~114.05 KB/116790, heapSize ~299.53 KB/306720, currentSize=87.22 KB/89310 for 7c7f87a6124bfe2f4049ee7803e0f98c in 890ms, sequenceid=351, compaction requested=true 2024-11-16T09:58:51,906 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 7c7f87a6124bfe2f4049ee7803e0f98c: 2024-11-16T09:58:51,906 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 7c7f87a6124bfe2f4049ee7803e0f98c:A, priority=-2147483648, current under compaction store size is 1 2024-11-16T09:58:51,906 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-16T09:58:51,906 DEBUG [RS:0;037c58a19ddb:37267-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-16T09:58:51,906 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 7c7f87a6124bfe2f4049ee7803e0f98c:B, priority=-2147483648, current under compaction store size is 2 2024-11-16T09:58:51,907 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-16T09:58:51,907 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 7c7f87a6124bfe2f4049ee7803e0f98c:C, priority=-2147483648, current under compaction store size is 3 2024-11-16T09:58:51,907 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-16T09:58:51,907 DEBUG [RS:0;037c58a19ddb:37267-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-16T09:58:51,909 DEBUG [RS:0;037c58a19ddb:37267-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37619 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-16T09:58:51,909 DEBUG [RS:0;037c58a19ddb:37267-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 103175 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-16T09:58:51,909 DEBUG [RS:0;037c58a19ddb:37267-longCompactions-0 {}] regionserver.HStore(1541): 7c7f87a6124bfe2f4049ee7803e0f98c/B is initiating minor compaction (all files) 2024-11-16T09:58:51,909 DEBUG [RS:0;037c58a19ddb:37267-shortCompactions-0 {}] regionserver.HStore(1541): 7c7f87a6124bfe2f4049ee7803e0f98c/A is initiating minor compaction (all files) 2024-11-16T09:58:51,909 INFO [RS:0;037c58a19ddb:37267-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 7c7f87a6124bfe2f4049ee7803e0f98c/B in TestAcidGuarantees,,1731751108989.7c7f87a6124bfe2f4049ee7803e0f98c. 2024-11-16T09:58:51,909 INFO [RS:0;037c58a19ddb:37267-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 7c7f87a6124bfe2f4049ee7803e0f98c/A in TestAcidGuarantees,,1731751108989.7c7f87a6124bfe2f4049ee7803e0f98c. 2024-11-16T09:58:51,909 INFO [RS:0;037c58a19ddb:37267-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/7c7f87a6124bfe2f4049ee7803e0f98c/B/e21881a409cd409893e1543bb8f3f4d3, hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/7c7f87a6124bfe2f4049ee7803e0f98c/B/0f9b87aa526f4d468458a3048268b749, hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/7c7f87a6124bfe2f4049ee7803e0f98c/B/96fd5ab09a8e47cba98b0b1f2bcda7a3] into tmpdir=hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/7c7f87a6124bfe2f4049ee7803e0f98c/.tmp, totalSize=36.7 K 2024-11-16T09:58:51,909 INFO [RS:0;037c58a19ddb:37267-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/7c7f87a6124bfe2f4049ee7803e0f98c/A/4ffc1030f8104b6ba06470b62b7108f6, hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/7c7f87a6124bfe2f4049ee7803e0f98c/A/76d20ead6b9f46678cadd75d8a84ae44, hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/7c7f87a6124bfe2f4049ee7803e0f98c/A/d3490b0bbc89425ca64d2102a813fa1f] into tmpdir=hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/7c7f87a6124bfe2f4049ee7803e0f98c/.tmp, totalSize=100.8 K 2024-11-16T09:58:51,909 INFO [RS:0;037c58a19ddb:37267-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731751108989.7c7f87a6124bfe2f4049ee7803e0f98c. 2024-11-16T09:58:51,909 DEBUG [RS:0;037c58a19ddb:37267-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731751108989.7c7f87a6124bfe2f4049ee7803e0f98c. files: [hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/7c7f87a6124bfe2f4049ee7803e0f98c/A/4ffc1030f8104b6ba06470b62b7108f6, hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/7c7f87a6124bfe2f4049ee7803e0f98c/A/76d20ead6b9f46678cadd75d8a84ae44, hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/7c7f87a6124bfe2f4049ee7803e0f98c/A/d3490b0bbc89425ca64d2102a813fa1f] 2024-11-16T09:58:51,909 DEBUG [RS:0;037c58a19ddb:37267-longCompactions-0 {}] compactions.Compactor(225): Compacting e21881a409cd409893e1543bb8f3f4d3, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=311, earliestPutTs=1731751126438 2024-11-16T09:58:51,910 DEBUG [RS:0;037c58a19ddb:37267-shortCompactions-0 {}] compactions.Compactor(225): Compacting 4ffc1030f8104b6ba06470b62b7108f6, keycount=150, bloomtype=ROW, size=31.2 K, encoding=NONE, compression=NONE, seqNum=311, earliestPutTs=1731751126438 2024-11-16T09:58:51,910 DEBUG [RS:0;037c58a19ddb:37267-longCompactions-0 {}] compactions.Compactor(225): Compacting 0f9b87aa526f4d468458a3048268b749, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=328, earliestPutTs=1731751127615 2024-11-16T09:58:51,910 DEBUG [RS:0;037c58a19ddb:37267-shortCompactions-0 {}] compactions.Compactor(225): Compacting 76d20ead6b9f46678cadd75d8a84ae44, keycount=150, bloomtype=ROW, size=30.5 K, encoding=NONE, compression=NONE, seqNum=328, earliestPutTs=1731751127615 2024-11-16T09:58:51,910 DEBUG [RS:0;037c58a19ddb:37267-longCompactions-0 {}] compactions.Compactor(225): Compacting 96fd5ab09a8e47cba98b0b1f2bcda7a3, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=351, earliestPutTs=1731751129871 2024-11-16T09:58:51,911 DEBUG [RS:0;037c58a19ddb:37267-shortCompactions-0 {}] compactions.Compactor(225): Compacting d3490b0bbc89425ca64d2102a813fa1f, keycount=200, bloomtype=ROW, size=39.0 K, encoding=NONE, compression=NONE, seqNum=351, earliestPutTs=1731751129861 2024-11-16T09:58:51,919 INFO [RS:0;037c58a19ddb:37267-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=7c7f87a6124bfe2f4049ee7803e0f98c] 2024-11-16T09:58:51,920 INFO [RS:0;037c58a19ddb:37267-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 7c7f87a6124bfe2f4049ee7803e0f98c#B#compaction#201 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-16T09:58:51,921 DEBUG [RS:0;037c58a19ddb:37267-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/7c7f87a6124bfe2f4049ee7803e0f98c/.tmp/B/0ff4b870412d4c84b679d600e6df9d98 is 50, key is test_row_0/B:col10/1731751129871/Put/seqid=0 2024-11-16T09:58:51,924 DEBUG [RS:0;037c58a19ddb:37267-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e2024111684a83252ca7746da862cc93b5eef3137_7c7f87a6124bfe2f4049ee7803e0f98c store=[table=TestAcidGuarantees family=A region=7c7f87a6124bfe2f4049ee7803e0f98c] 2024-11-16T09:58:51,926 DEBUG [RS:0;037c58a19ddb:37267-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e2024111684a83252ca7746da862cc93b5eef3137_7c7f87a6124bfe2f4049ee7803e0f98c, store=[table=TestAcidGuarantees family=A region=7c7f87a6124bfe2f4049ee7803e0f98c] 2024-11-16T09:58:51,926 DEBUG [RS:0;037c58a19ddb:37267-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111684a83252ca7746da862cc93b5eef3137_7c7f87a6124bfe2f4049ee7803e0f98c because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=7c7f87a6124bfe2f4049ee7803e0f98c] 2024-11-16T09:58:51,938 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37591 is added to blk_1073742065_1241 (size=13119) 2024-11-16T09:58:51,946 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37591 is added to blk_1073742066_1242 (size=4469) 2024-11-16T09:58:51,947 INFO [RS:0;037c58a19ddb:37267-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 7c7f87a6124bfe2f4049ee7803e0f98c#A#compaction#202 average throughput is 0.87 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-16T09:58:51,948 DEBUG [RS:0;037c58a19ddb:37267-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/7c7f87a6124bfe2f4049ee7803e0f98c/.tmp/A/4e45d1dfc3424857bed4a7303fd573eb is 175, key is test_row_0/A:col10/1731751129871/Put/seqid=0 2024-11-16T09:58:51,960 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37591 is added to blk_1073742067_1243 (size=32073) 2024-11-16T09:58:51,967 DEBUG [RS:0;037c58a19ddb:37267-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/7c7f87a6124bfe2f4049ee7803e0f98c/.tmp/A/4e45d1dfc3424857bed4a7303fd573eb as hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/7c7f87a6124bfe2f4049ee7803e0f98c/A/4e45d1dfc3424857bed4a7303fd573eb 2024-11-16T09:58:51,972 INFO [RS:0;037c58a19ddb:37267-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 7c7f87a6124bfe2f4049ee7803e0f98c/A of 7c7f87a6124bfe2f4049ee7803e0f98c into 4e45d1dfc3424857bed4a7303fd573eb(size=31.3 K), total size for store is 31.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-16T09:58:51,972 DEBUG [RS:0;037c58a19ddb:37267-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 7c7f87a6124bfe2f4049ee7803e0f98c: 2024-11-16T09:58:51,972 INFO [RS:0;037c58a19ddb:37267-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731751108989.7c7f87a6124bfe2f4049ee7803e0f98c., storeName=7c7f87a6124bfe2f4049ee7803e0f98c/A, priority=13, startTime=1731751131906; duration=0sec 2024-11-16T09:58:51,972 DEBUG [RS:0;037c58a19ddb:37267-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-16T09:58:51,972 DEBUG [RS:0;037c58a19ddb:37267-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 7c7f87a6124bfe2f4049ee7803e0f98c:A 2024-11-16T09:58:51,972 DEBUG [RS:0;037c58a19ddb:37267-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-16T09:58:51,974 DEBUG [RS:0;037c58a19ddb:37267-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37619 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-16T09:58:51,974 DEBUG [RS:0;037c58a19ddb:37267-shortCompactions-0 {}] regionserver.HStore(1541): 7c7f87a6124bfe2f4049ee7803e0f98c/C is initiating minor compaction (all files) 2024-11-16T09:58:51,974 INFO [RS:0;037c58a19ddb:37267-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 7c7f87a6124bfe2f4049ee7803e0f98c/C in TestAcidGuarantees,,1731751108989.7c7f87a6124bfe2f4049ee7803e0f98c. 2024-11-16T09:58:51,974 INFO [RS:0;037c58a19ddb:37267-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/7c7f87a6124bfe2f4049ee7803e0f98c/C/713ae3ec75aa419e8acd7ec473839cc8, hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/7c7f87a6124bfe2f4049ee7803e0f98c/C/9b4c5b55ab9a47fb97b7f974d882af88, hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/7c7f87a6124bfe2f4049ee7803e0f98c/C/745a570470de439d839401a55fbe1de1] into tmpdir=hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/7c7f87a6124bfe2f4049ee7803e0f98c/.tmp, totalSize=36.7 K 2024-11-16T09:58:51,976 DEBUG [RS:0;037c58a19ddb:37267-shortCompactions-0 {}] compactions.Compactor(225): Compacting 713ae3ec75aa419e8acd7ec473839cc8, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=311, earliestPutTs=1731751126438 2024-11-16T09:58:51,976 DEBUG [RS:0;037c58a19ddb:37267-shortCompactions-0 {}] compactions.Compactor(225): Compacting 9b4c5b55ab9a47fb97b7f974d882af88, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=328, earliestPutTs=1731751127615 2024-11-16T09:58:51,976 DEBUG [RS:0;037c58a19ddb:37267-shortCompactions-0 {}] compactions.Compactor(225): Compacting 745a570470de439d839401a55fbe1de1, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=351, earliestPutTs=1731751129871 2024-11-16T09:58:51,985 INFO [RS:0;037c58a19ddb:37267-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 7c7f87a6124bfe2f4049ee7803e0f98c#C#compaction#203 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-16T09:58:51,985 DEBUG [RS:0;037c58a19ddb:37267-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/7c7f87a6124bfe2f4049ee7803e0f98c/.tmp/C/1992ed43352e4e4aba1fb223c6698799 is 50, key is test_row_0/C:col10/1731751129871/Put/seqid=0 2024-11-16T09:58:51,997 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37591 is added to blk_1073742068_1244 (size=13119) 2024-11-16T09:58:52,003 DEBUG [RS:0;037c58a19ddb:37267-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/7c7f87a6124bfe2f4049ee7803e0f98c/.tmp/C/1992ed43352e4e4aba1fb223c6698799 as hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/7c7f87a6124bfe2f4049ee7803e0f98c/C/1992ed43352e4e4aba1fb223c6698799 2024-11-16T09:58:52,010 INFO [RS:0;037c58a19ddb:37267-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 7c7f87a6124bfe2f4049ee7803e0f98c/C of 7c7f87a6124bfe2f4049ee7803e0f98c into 1992ed43352e4e4aba1fb223c6698799(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-16T09:58:52,010 DEBUG [RS:0;037c58a19ddb:37267-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 7c7f87a6124bfe2f4049ee7803e0f98c: 2024-11-16T09:58:52,010 INFO [RS:0;037c58a19ddb:37267-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731751108989.7c7f87a6124bfe2f4049ee7803e0f98c., storeName=7c7f87a6124bfe2f4049ee7803e0f98c/C, priority=13, startTime=1731751131907; duration=0sec 2024-11-16T09:58:52,010 DEBUG [RS:0;037c58a19ddb:37267-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-16T09:58:52,010 DEBUG [RS:0;037c58a19ddb:37267-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 7c7f87a6124bfe2f4049ee7803e0f98c:C 2024-11-16T09:58:52,024 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=37267 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=51 2024-11-16T09:58:52,024 DEBUG [RS_FLUSH_OPERATIONS-regionserver/037c58a19ddb:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731751108989.7c7f87a6124bfe2f4049ee7803e0f98c. 2024-11-16T09:58:52,024 INFO [RS_FLUSH_OPERATIONS-regionserver/037c58a19ddb:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HRegion(2902): Flushing 7c7f87a6124bfe2f4049ee7803e0f98c 3/3 column families, dataSize=87.22 KB heapSize=229.27 KB 2024-11-16T09:58:52,033 DEBUG [RS_FLUSH_OPERATIONS-regionserver/037c58a19ddb:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241116a4bd5237f1004a00ae64e4d510126295_7c7f87a6124bfe2f4049ee7803e0f98c is 50, key is test_row_0/A:col10/1731751131072/Put/seqid=0 2024-11-16T09:58:52,041 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37591 is added to blk_1073742069_1245 (size=12454) 2024-11-16T09:58:52,183 INFO [Thread-720 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-720. 2024-11-16T09:58:52,183 DEBUG [Thread-720 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-16T09:58:52,183 DEBUG [Thread-720 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T09:58:52,183 DEBUG [Thread-720 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T09:58:52,184 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-16T09:58:52,184 INFO [Thread-718 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-718. 2024-11-16T09:58:52,184 DEBUG [Thread-718 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-16T09:58:52,184 DEBUG [Thread-718 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T09:58:52,184 DEBUG [Thread-718 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T09:58:52,184 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-16T09:58:52,185 INFO [Thread-724 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-724. 2024-11-16T09:58:52,185 DEBUG [Thread-724 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-16T09:58:52,185 DEBUG [Thread-724 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T09:58:52,185 DEBUG [Thread-724 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T09:58:52,185 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-16T09:58:52,188 INFO [Thread-722 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-722. 2024-11-16T09:58:52,188 DEBUG [Thread-722 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-16T09:58:52,188 DEBUG [Thread-722 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T09:58:52,188 DEBUG [Thread-722 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T09:58:52,188 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-16T09:58:52,214 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] regionserver.HRegion(8855): Flush requested on 7c7f87a6124bfe2f4049ee7803e0f98c 2024-11-16T09:58:52,214 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731751108989.7c7f87a6124bfe2f4049ee7803e0f98c. as already flushing 2024-11-16T09:58:52,214 INFO [Thread-715 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-715. 2024-11-16T09:58:52,214 DEBUG [Thread-715 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-16T09:58:52,214 DEBUG [Thread-715 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T09:58:52,214 DEBUG [Thread-715 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T09:58:52,214 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-16T09:58:52,242 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37877 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=50 2024-11-16T09:58:52,343 DEBUG [RS:0;037c58a19ddb:37267-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/7c7f87a6124bfe2f4049ee7803e0f98c/.tmp/B/0ff4b870412d4c84b679d600e6df9d98 as hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/7c7f87a6124bfe2f4049ee7803e0f98c/B/0ff4b870412d4c84b679d600e6df9d98 2024-11-16T09:58:52,348 INFO [RS:0;037c58a19ddb:37267-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 7c7f87a6124bfe2f4049ee7803e0f98c/B of 7c7f87a6124bfe2f4049ee7803e0f98c into 0ff4b870412d4c84b679d600e6df9d98(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-16T09:58:52,348 DEBUG [RS:0;037c58a19ddb:37267-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 7c7f87a6124bfe2f4049ee7803e0f98c: 2024-11-16T09:58:52,348 INFO [RS:0;037c58a19ddb:37267-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731751108989.7c7f87a6124bfe2f4049ee7803e0f98c., storeName=7c7f87a6124bfe2f4049ee7803e0f98c/B, priority=13, startTime=1731751131906; duration=0sec 2024-11-16T09:58:52,348 DEBUG [RS:0;037c58a19ddb:37267-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-16T09:58:52,348 DEBUG [RS:0;037c58a19ddb:37267-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 7c7f87a6124bfe2f4049ee7803e0f98c:B 2024-11-16T09:58:52,442 DEBUG [RS_FLUSH_OPERATIONS-regionserver/037c58a19ddb:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T09:58:52,446 INFO [RS_FLUSH_OPERATIONS-regionserver/037c58a19ddb:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241116a4bd5237f1004a00ae64e4d510126295_7c7f87a6124bfe2f4049ee7803e0f98c to hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241116a4bd5237f1004a00ae64e4d510126295_7c7f87a6124bfe2f4049ee7803e0f98c 2024-11-16T09:58:52,446 DEBUG [RS_FLUSH_OPERATIONS-regionserver/037c58a19ddb:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/7c7f87a6124bfe2f4049ee7803e0f98c/.tmp/A/b136583a9ba84d67a51c93bf74a272b9, store: [table=TestAcidGuarantees family=A region=7c7f87a6124bfe2f4049ee7803e0f98c] 2024-11-16T09:58:52,447 DEBUG [RS_FLUSH_OPERATIONS-regionserver/037c58a19ddb:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/7c7f87a6124bfe2f4049ee7803e0f98c/.tmp/A/b136583a9ba84d67a51c93bf74a272b9 is 175, key is test_row_0/A:col10/1731751131072/Put/seqid=0 2024-11-16T09:58:52,450 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37591 is added to blk_1073742070_1246 (size=31255) 2024-11-16T09:58:52,564 INFO [Thread-713 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-713. 2024-11-16T09:58:52,564 DEBUG [Thread-713 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-16T09:58:52,564 DEBUG [Thread-713 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T09:58:52,564 DEBUG [Thread-713 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T09:58:52,564 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-16T09:58:52,584 INFO [Thread-711 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-711. 2024-11-16T09:58:52,584 DEBUG [Thread-711 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-16T09:58:52,584 DEBUG [Thread-711 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T09:58:52,584 DEBUG [Thread-711 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T09:58:52,584 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-16T09:58:52,594 INFO [Thread-709 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-709. 2024-11-16T09:58:52,594 INFO [Thread-707 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-707. 2024-11-16T09:58:52,594 DEBUG [Thread-709 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-16T09:58:52,594 DEBUG [Thread-709 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T09:58:52,594 DEBUG [Thread-707 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-16T09:58:52,594 DEBUG [Thread-709 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T09:58:52,594 DEBUG [Thread-707 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T09:58:52,594 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-16T09:58:52,595 DEBUG [Thread-707 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T09:58:52,595 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-16T09:58:52,851 INFO [RS_FLUSH_OPERATIONS-regionserver/037c58a19ddb:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=369, memsize=29.1 K, hasBloomFilter=true, into tmp file hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/7c7f87a6124bfe2f4049ee7803e0f98c/.tmp/A/b136583a9ba84d67a51c93bf74a272b9 2024-11-16T09:58:52,858 DEBUG [RS_FLUSH_OPERATIONS-regionserver/037c58a19ddb:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/7c7f87a6124bfe2f4049ee7803e0f98c/.tmp/B/825d9c7704f0471f940babf25293601e is 50, key is test_row_0/B:col10/1731751131072/Put/seqid=0 2024-11-16T09:58:52,861 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37591 is added to blk_1073742071_1247 (size=12301) 2024-11-16T09:58:53,252 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37877 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=50 2024-11-16T09:58:53,262 INFO [RS_FLUSH_OPERATIONS-regionserver/037c58a19ddb:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=369 (bloomFilter=true), to=hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/7c7f87a6124bfe2f4049ee7803e0f98c/.tmp/B/825d9c7704f0471f940babf25293601e 2024-11-16T09:58:53,269 DEBUG [RS_FLUSH_OPERATIONS-regionserver/037c58a19ddb:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/7c7f87a6124bfe2f4049ee7803e0f98c/.tmp/C/f621f228b55141e4abbbe1ee9bd02957 is 50, key is test_row_0/C:col10/1731751131072/Put/seqid=0 2024-11-16T09:58:53,272 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37591 is added to blk_1073742072_1248 (size=12301) 2024-11-16T09:58:53,673 INFO [RS_FLUSH_OPERATIONS-regionserver/037c58a19ddb:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=369 (bloomFilter=true), to=hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/7c7f87a6124bfe2f4049ee7803e0f98c/.tmp/C/f621f228b55141e4abbbe1ee9bd02957 2024-11-16T09:58:53,678 DEBUG [RS_FLUSH_OPERATIONS-regionserver/037c58a19ddb:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/7c7f87a6124bfe2f4049ee7803e0f98c/.tmp/A/b136583a9ba84d67a51c93bf74a272b9 as hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/7c7f87a6124bfe2f4049ee7803e0f98c/A/b136583a9ba84d67a51c93bf74a272b9 2024-11-16T09:58:53,682 INFO [RS_FLUSH_OPERATIONS-regionserver/037c58a19ddb:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/7c7f87a6124bfe2f4049ee7803e0f98c/A/b136583a9ba84d67a51c93bf74a272b9, entries=150, sequenceid=369, filesize=30.5 K 2024-11-16T09:58:53,683 DEBUG [RS_FLUSH_OPERATIONS-regionserver/037c58a19ddb:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/7c7f87a6124bfe2f4049ee7803e0f98c/.tmp/B/825d9c7704f0471f940babf25293601e as hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/7c7f87a6124bfe2f4049ee7803e0f98c/B/825d9c7704f0471f940babf25293601e 2024-11-16T09:58:53,687 INFO [RS_FLUSH_OPERATIONS-regionserver/037c58a19ddb:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/7c7f87a6124bfe2f4049ee7803e0f98c/B/825d9c7704f0471f940babf25293601e, entries=150, sequenceid=369, filesize=12.0 K 2024-11-16T09:58:53,688 DEBUG [RS_FLUSH_OPERATIONS-regionserver/037c58a19ddb:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/7c7f87a6124bfe2f4049ee7803e0f98c/.tmp/C/f621f228b55141e4abbbe1ee9bd02957 as hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/7c7f87a6124bfe2f4049ee7803e0f98c/C/f621f228b55141e4abbbe1ee9bd02957 2024-11-16T09:58:53,691 INFO [RS_FLUSH_OPERATIONS-regionserver/037c58a19ddb:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/7c7f87a6124bfe2f4049ee7803e0f98c/C/f621f228b55141e4abbbe1ee9bd02957, entries=150, sequenceid=369, filesize=12.0 K 2024-11-16T09:58:53,692 INFO [RS_FLUSH_OPERATIONS-regionserver/037c58a19ddb:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HRegion(3140): Finished flush of dataSize ~87.22 KB/89310, heapSize ~229.22 KB/234720, currentSize=33.54 KB/34350 for 7c7f87a6124bfe2f4049ee7803e0f98c in 1668ms, sequenceid=369, compaction requested=false 2024-11-16T09:58:53,692 DEBUG [RS_FLUSH_OPERATIONS-regionserver/037c58a19ddb:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HRegion(2603): Flush status journal for 7c7f87a6124bfe2f4049ee7803e0f98c: 2024-11-16T09:58:53,692 DEBUG [RS_FLUSH_OPERATIONS-regionserver/037c58a19ddb:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731751108989.7c7f87a6124bfe2f4049ee7803e0f98c. 2024-11-16T09:58:53,692 DEBUG [RS_FLUSH_OPERATIONS-regionserver/037c58a19ddb:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=51 2024-11-16T09:58:53,692 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37877 {}] master.HMaster(4169): Remote procedure done, pid=51 2024-11-16T09:58:53,695 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=51, resume processing ppid=50 2024-11-16T09:58:53,695 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=51, ppid=50, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.5870 sec 2024-11-16T09:58:53,697 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=50, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=50, table=TestAcidGuarantees in 2.5920 sec 2024-11-16T09:58:55,273 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37877 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=50 2024-11-16T09:58:55,273 INFO [RPCClient-NioEventLoopGroup-4-7 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-16T09:58:55,273 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(392): Finished test. Writers: 2024-11-16T09:58:55,273 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 47 2024-11-16T09:58:55,273 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 51 2024-11-16T09:58:55,273 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 59 2024-11-16T09:58:55,273 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 56 2024-11-16T09:58:55,273 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 80 2024-11-16T09:58:55,273 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(396): Readers: 2024-11-16T09:58:55,273 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 3479 2024-11-16T09:58:55,273 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 3319 2024-11-16T09:58:55,273 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(400): Scanners: 2024-11-16T09:58:55,273 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1509 2024-11-16T09:58:55,273 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 4525 rows 2024-11-16T09:58:55,273 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1509 2024-11-16T09:58:55,273 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 4525 rows 2024-11-16T09:58:55,273 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-16T09:58:55,273 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool.doWork(AcidGuaranteesTestTool.java:131) at org.apache.hadoop.hbase.util.AbstractHBaseTool.run(AbstractHBaseTool.java:150) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.runTestAtomicity(AcidGuaranteesTestBase.java:102) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.testMobMixedAtomicity(AcidGuaranteesTestBase.java:132) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-16T09:58:55,273 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T09:58:55,274 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T09:58:55,274 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-16T09:58:55,274 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37877 {}] master.HMaster$13(2820): Client=jenkins//172.17.0.2 disable TestAcidGuarantees 2024-11-16T09:58:55,275 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37877 {}] procedure2.ProcedureExecutor(1139): Stored pid=52, state=RUNNABLE:DISABLE_TABLE_PREPARE, hasLock=false; DisableTableProcedure table=TestAcidGuarantees 2024-11-16T09:58:55,277 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37877 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=52 2024-11-16T09:58:55,277 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731751135277"}]},"ts":"1731751135277"} 2024-11-16T09:58:55,279 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLING in hbase:meta 2024-11-16T09:58:55,279 INFO [PEWorker-3 {}] procedure.DisableTableProcedure(284): Set TestAcidGuarantees to state=DISABLING 2024-11-16T09:58:55,279 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=53, ppid=52, state=RUNNABLE:CLOSE_TABLE_REGIONS_SCHEDULE, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees}] 2024-11-16T09:58:55,281 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=54, ppid=53, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=7c7f87a6124bfe2f4049ee7803e0f98c, UNASSIGN}] 2024-11-16T09:58:55,281 INFO [PEWorker-4 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=54, ppid=53, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=7c7f87a6124bfe2f4049ee7803e0f98c, UNASSIGN 2024-11-16T09:58:55,282 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=54 updating hbase:meta row=7c7f87a6124bfe2f4049ee7803e0f98c, regionState=CLOSING, regionLocation=037c58a19ddb,37267,1731751077844 2024-11-16T09:58:55,283 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=54, ppid=53, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=7c7f87a6124bfe2f4049ee7803e0f98c, UNASSIGN because future has completed 2024-11-16T09:58:55,283 DEBUG [PEWorker-1 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-11-16T09:58:55,283 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=55, ppid=54, state=RUNNABLE, hasLock=false; CloseRegionProcedure 7c7f87a6124bfe2f4049ee7803e0f98c, server=037c58a19ddb,37267,1731751077844}] 2024-11-16T09:58:55,382 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37877 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=52 2024-11-16T09:58:55,435 INFO [RS_CLOSE_REGION-regionserver/037c58a19ddb:0-0 {event_type=M_RS_CLOSE_REGION, pid=55}] handler.UnassignRegionHandler(122): Close 7c7f87a6124bfe2f4049ee7803e0f98c 2024-11-16T09:58:55,435 DEBUG [RS_CLOSE_REGION-regionserver/037c58a19ddb:0-0 {event_type=M_RS_CLOSE_REGION, pid=55}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-11-16T09:58:55,435 DEBUG [RS_CLOSE_REGION-regionserver/037c58a19ddb:0-0 {event_type=M_RS_CLOSE_REGION, pid=55}] regionserver.HRegion(1722): Closing 7c7f87a6124bfe2f4049ee7803e0f98c, disabling compactions & flushes 2024-11-16T09:58:55,436 INFO [RS_CLOSE_REGION-regionserver/037c58a19ddb:0-0 {event_type=M_RS_CLOSE_REGION, pid=55}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1731751108989.7c7f87a6124bfe2f4049ee7803e0f98c. 2024-11-16T09:58:55,436 DEBUG [RS_CLOSE_REGION-regionserver/037c58a19ddb:0-0 {event_type=M_RS_CLOSE_REGION, pid=55}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1731751108989.7c7f87a6124bfe2f4049ee7803e0f98c. 2024-11-16T09:58:55,436 DEBUG [RS_CLOSE_REGION-regionserver/037c58a19ddb:0-0 {event_type=M_RS_CLOSE_REGION, pid=55}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1731751108989.7c7f87a6124bfe2f4049ee7803e0f98c. after waiting 0 ms 2024-11-16T09:58:55,436 DEBUG [RS_CLOSE_REGION-regionserver/037c58a19ddb:0-0 {event_type=M_RS_CLOSE_REGION, pid=55}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1731751108989.7c7f87a6124bfe2f4049ee7803e0f98c. 2024-11-16T09:58:55,436 INFO [RS_CLOSE_REGION-regionserver/037c58a19ddb:0-0 {event_type=M_RS_CLOSE_REGION, pid=55}] regionserver.HRegion(2902): Flushing 7c7f87a6124bfe2f4049ee7803e0f98c 3/3 column families, dataSize=33.54 KB heapSize=88.64 KB 2024-11-16T09:58:55,442 DEBUG [RS_CLOSE_REGION-regionserver/037c58a19ddb:0-0 {event_type=M_RS_CLOSE_REGION, pid=55}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241116f5691975dc39470781ff8c987c0076d4_7c7f87a6124bfe2f4049ee7803e0f98c is 50, key is test_row_0/A:col10/1731751132213/Put/seqid=0 2024-11-16T09:58:55,446 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37591 is added to blk_1073742073_1249 (size=12454) 2024-11-16T09:58:55,593 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37877 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=52 2024-11-16T09:58:55,846 DEBUG [RS_CLOSE_REGION-regionserver/037c58a19ddb:0-0 {event_type=M_RS_CLOSE_REGION, pid=55}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T09:58:55,850 INFO [RS_CLOSE_REGION-regionserver/037c58a19ddb:0-0 {event_type=M_RS_CLOSE_REGION, pid=55}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241116f5691975dc39470781ff8c987c0076d4_7c7f87a6124bfe2f4049ee7803e0f98c to hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241116f5691975dc39470781ff8c987c0076d4_7c7f87a6124bfe2f4049ee7803e0f98c 2024-11-16T09:58:55,851 DEBUG [RS_CLOSE_REGION-regionserver/037c58a19ddb:0-0 {event_type=M_RS_CLOSE_REGION, pid=55}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/7c7f87a6124bfe2f4049ee7803e0f98c/.tmp/A/3a58c36cb1cb4b19a02cd633e4f31471, store: [table=TestAcidGuarantees family=A region=7c7f87a6124bfe2f4049ee7803e0f98c] 2024-11-16T09:58:55,852 DEBUG [RS_CLOSE_REGION-regionserver/037c58a19ddb:0-0 {event_type=M_RS_CLOSE_REGION, pid=55}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/7c7f87a6124bfe2f4049ee7803e0f98c/.tmp/A/3a58c36cb1cb4b19a02cd633e4f31471 is 175, key is test_row_0/A:col10/1731751132213/Put/seqid=0 2024-11-16T09:58:55,855 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37591 is added to blk_1073742074_1250 (size=31255) 2024-11-16T09:58:55,903 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37877 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=52 2024-11-16T09:58:56,235 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtil$FsDatasetAsyncDiskServiceFixer(597): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details. 2024-11-16T09:58:56,256 INFO [RS_CLOSE_REGION-regionserver/037c58a19ddb:0-0 {event_type=M_RS_CLOSE_REGION, pid=55}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=378, memsize=11.2 K, hasBloomFilter=true, into tmp file hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/7c7f87a6124bfe2f4049ee7803e0f98c/.tmp/A/3a58c36cb1cb4b19a02cd633e4f31471 2024-11-16T09:58:56,263 DEBUG [RS_CLOSE_REGION-regionserver/037c58a19ddb:0-0 {event_type=M_RS_CLOSE_REGION, pid=55}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/7c7f87a6124bfe2f4049ee7803e0f98c/.tmp/B/5ea8a2c65fcb4e1abf1189f016dde782 is 50, key is test_row_0/B:col10/1731751132213/Put/seqid=0 2024-11-16T09:58:56,266 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37591 is added to blk_1073742075_1251 (size=12301) 2024-11-16T09:58:56,412 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37877 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=52 2024-11-16T09:58:56,667 INFO [RS_CLOSE_REGION-regionserver/037c58a19ddb:0-0 {event_type=M_RS_CLOSE_REGION, pid=55}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=378 (bloomFilter=true), to=hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/7c7f87a6124bfe2f4049ee7803e0f98c/.tmp/B/5ea8a2c65fcb4e1abf1189f016dde782 2024-11-16T09:58:56,674 DEBUG [RS_CLOSE_REGION-regionserver/037c58a19ddb:0-0 {event_type=M_RS_CLOSE_REGION, pid=55}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/7c7f87a6124bfe2f4049ee7803e0f98c/.tmp/C/3d73c76130644d799940328ba53420cd is 50, key is test_row_0/C:col10/1731751132213/Put/seqid=0 2024-11-16T09:58:56,677 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37591 is added to blk_1073742076_1252 (size=12301) 2024-11-16T09:58:57,078 INFO [RS_CLOSE_REGION-regionserver/037c58a19ddb:0-0 {event_type=M_RS_CLOSE_REGION, pid=55}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=378 (bloomFilter=true), to=hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/7c7f87a6124bfe2f4049ee7803e0f98c/.tmp/C/3d73c76130644d799940328ba53420cd 2024-11-16T09:58:57,082 DEBUG [RS_CLOSE_REGION-regionserver/037c58a19ddb:0-0 {event_type=M_RS_CLOSE_REGION, pid=55}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/7c7f87a6124bfe2f4049ee7803e0f98c/.tmp/A/3a58c36cb1cb4b19a02cd633e4f31471 as hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/7c7f87a6124bfe2f4049ee7803e0f98c/A/3a58c36cb1cb4b19a02cd633e4f31471 2024-11-16T09:58:57,086 INFO [RS_CLOSE_REGION-regionserver/037c58a19ddb:0-0 {event_type=M_RS_CLOSE_REGION, pid=55}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/7c7f87a6124bfe2f4049ee7803e0f98c/A/3a58c36cb1cb4b19a02cd633e4f31471, entries=150, sequenceid=378, filesize=30.5 K 2024-11-16T09:58:57,086 DEBUG [RS_CLOSE_REGION-regionserver/037c58a19ddb:0-0 {event_type=M_RS_CLOSE_REGION, pid=55}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/7c7f87a6124bfe2f4049ee7803e0f98c/.tmp/B/5ea8a2c65fcb4e1abf1189f016dde782 as hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/7c7f87a6124bfe2f4049ee7803e0f98c/B/5ea8a2c65fcb4e1abf1189f016dde782 2024-11-16T09:58:57,089 INFO [RS_CLOSE_REGION-regionserver/037c58a19ddb:0-0 {event_type=M_RS_CLOSE_REGION, pid=55}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/7c7f87a6124bfe2f4049ee7803e0f98c/B/5ea8a2c65fcb4e1abf1189f016dde782, entries=150, sequenceid=378, filesize=12.0 K 2024-11-16T09:58:57,090 DEBUG [RS_CLOSE_REGION-regionserver/037c58a19ddb:0-0 {event_type=M_RS_CLOSE_REGION, pid=55}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/7c7f87a6124bfe2f4049ee7803e0f98c/.tmp/C/3d73c76130644d799940328ba53420cd as hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/7c7f87a6124bfe2f4049ee7803e0f98c/C/3d73c76130644d799940328ba53420cd 2024-11-16T09:58:57,093 INFO [RS_CLOSE_REGION-regionserver/037c58a19ddb:0-0 {event_type=M_RS_CLOSE_REGION, pid=55}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/7c7f87a6124bfe2f4049ee7803e0f98c/C/3d73c76130644d799940328ba53420cd, entries=150, sequenceid=378, filesize=12.0 K 2024-11-16T09:58:57,094 INFO [RS_CLOSE_REGION-regionserver/037c58a19ddb:0-0 {event_type=M_RS_CLOSE_REGION, pid=55}] regionserver.HRegion(3140): Finished flush of dataSize ~33.54 KB/34350, heapSize ~88.59 KB/90720, currentSize=0 B/0 for 7c7f87a6124bfe2f4049ee7803e0f98c in 1658ms, sequenceid=378, compaction requested=true 2024-11-16T09:58:57,095 DEBUG [StoreCloser-TestAcidGuarantees,,1731751108989.7c7f87a6124bfe2f4049ee7803e0f98c.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/7c7f87a6124bfe2f4049ee7803e0f98c/A/c6a8bb2378f148b9ad055005e5046ab4, hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/7c7f87a6124bfe2f4049ee7803e0f98c/A/ee8a20ddfc1549a1b4d7045964ed8bb8, hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/7c7f87a6124bfe2f4049ee7803e0f98c/A/a9b0a7760e764325a134f8bd6fd9c882, hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/7c7f87a6124bfe2f4049ee7803e0f98c/A/8aaa28ea93ee462eaa017eee7d302c00, hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/7c7f87a6124bfe2f4049ee7803e0f98c/A/decd818193a64048853aa6c79bdf0976, hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/7c7f87a6124bfe2f4049ee7803e0f98c/A/d4eba6854cf24b93ab8094b32b349735, hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/7c7f87a6124bfe2f4049ee7803e0f98c/A/d69fd5bf934e49ad99e55960bda383ae, hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/7c7f87a6124bfe2f4049ee7803e0f98c/A/4ca189b9a842483db6e475afcc8899aa, hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/7c7f87a6124bfe2f4049ee7803e0f98c/A/64fcc2a7d41e439a952ea7e0969d6e41, hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/7c7f87a6124bfe2f4049ee7803e0f98c/A/8c843a2fb4074619b6de587720d98901, hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/7c7f87a6124bfe2f4049ee7803e0f98c/A/2dddce9f3cd147d3b9cd2f2d801fc58a, hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/7c7f87a6124bfe2f4049ee7803e0f98c/A/3bff852129eb44848826f920130a1069, hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/7c7f87a6124bfe2f4049ee7803e0f98c/A/42e604f249434caf82bcc0dc3532e984, hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/7c7f87a6124bfe2f4049ee7803e0f98c/A/6b3a4c4820f0410ba008ad14ccf8af07, hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/7c7f87a6124bfe2f4049ee7803e0f98c/A/9db025a49e5b45598af87c7bf658d34b, hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/7c7f87a6124bfe2f4049ee7803e0f98c/A/2049d15304474f38af50e9ff363d240f, hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/7c7f87a6124bfe2f4049ee7803e0f98c/A/15c9a113751c448ab4ea0ce9821776a1, hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/7c7f87a6124bfe2f4049ee7803e0f98c/A/36f2297c7adc4ecaaf1ca484ad85f45f, hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/7c7f87a6124bfe2f4049ee7803e0f98c/A/cf911fd937c34d99a23ab70841f56bfe, hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/7c7f87a6124bfe2f4049ee7803e0f98c/A/5ea7fb182fe64f0b9234c748529c1dec, hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/7c7f87a6124bfe2f4049ee7803e0f98c/A/34e9400bce784c0fa95be143ab8eefe9, hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/7c7f87a6124bfe2f4049ee7803e0f98c/A/4ffc1030f8104b6ba06470b62b7108f6, hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/7c7f87a6124bfe2f4049ee7803e0f98c/A/76d20ead6b9f46678cadd75d8a84ae44, hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/7c7f87a6124bfe2f4049ee7803e0f98c/A/d3490b0bbc89425ca64d2102a813fa1f] to archive 2024-11-16T09:58:57,096 DEBUG [StoreCloser-TestAcidGuarantees,,1731751108989.7c7f87a6124bfe2f4049ee7803e0f98c.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-16T09:58:57,097 DEBUG [StoreCloser-TestAcidGuarantees,,1731751108989.7c7f87a6124bfe2f4049ee7803e0f98c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/7c7f87a6124bfe2f4049ee7803e0f98c/A/c6a8bb2378f148b9ad055005e5046ab4 to hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/archive/data/default/TestAcidGuarantees/7c7f87a6124bfe2f4049ee7803e0f98c/A/c6a8bb2378f148b9ad055005e5046ab4 2024-11-16T09:58:57,098 DEBUG [StoreCloser-TestAcidGuarantees,,1731751108989.7c7f87a6124bfe2f4049ee7803e0f98c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/7c7f87a6124bfe2f4049ee7803e0f98c/A/ee8a20ddfc1549a1b4d7045964ed8bb8 to hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/archive/data/default/TestAcidGuarantees/7c7f87a6124bfe2f4049ee7803e0f98c/A/ee8a20ddfc1549a1b4d7045964ed8bb8 2024-11-16T09:58:57,099 DEBUG [StoreCloser-TestAcidGuarantees,,1731751108989.7c7f87a6124bfe2f4049ee7803e0f98c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/7c7f87a6124bfe2f4049ee7803e0f98c/A/a9b0a7760e764325a134f8bd6fd9c882 to hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/archive/data/default/TestAcidGuarantees/7c7f87a6124bfe2f4049ee7803e0f98c/A/a9b0a7760e764325a134f8bd6fd9c882 2024-11-16T09:58:57,101 DEBUG [StoreCloser-TestAcidGuarantees,,1731751108989.7c7f87a6124bfe2f4049ee7803e0f98c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/7c7f87a6124bfe2f4049ee7803e0f98c/A/8aaa28ea93ee462eaa017eee7d302c00 to hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/archive/data/default/TestAcidGuarantees/7c7f87a6124bfe2f4049ee7803e0f98c/A/8aaa28ea93ee462eaa017eee7d302c00 2024-11-16T09:58:57,102 DEBUG [StoreCloser-TestAcidGuarantees,,1731751108989.7c7f87a6124bfe2f4049ee7803e0f98c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/7c7f87a6124bfe2f4049ee7803e0f98c/A/decd818193a64048853aa6c79bdf0976 to hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/archive/data/default/TestAcidGuarantees/7c7f87a6124bfe2f4049ee7803e0f98c/A/decd818193a64048853aa6c79bdf0976 2024-11-16T09:58:57,102 DEBUG [StoreCloser-TestAcidGuarantees,,1731751108989.7c7f87a6124bfe2f4049ee7803e0f98c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/7c7f87a6124bfe2f4049ee7803e0f98c/A/d4eba6854cf24b93ab8094b32b349735 to hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/archive/data/default/TestAcidGuarantees/7c7f87a6124bfe2f4049ee7803e0f98c/A/d4eba6854cf24b93ab8094b32b349735 2024-11-16T09:58:57,103 DEBUG [StoreCloser-TestAcidGuarantees,,1731751108989.7c7f87a6124bfe2f4049ee7803e0f98c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/7c7f87a6124bfe2f4049ee7803e0f98c/A/d69fd5bf934e49ad99e55960bda383ae to hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/archive/data/default/TestAcidGuarantees/7c7f87a6124bfe2f4049ee7803e0f98c/A/d69fd5bf934e49ad99e55960bda383ae 2024-11-16T09:58:57,104 DEBUG [StoreCloser-TestAcidGuarantees,,1731751108989.7c7f87a6124bfe2f4049ee7803e0f98c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/7c7f87a6124bfe2f4049ee7803e0f98c/A/4ca189b9a842483db6e475afcc8899aa to hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/archive/data/default/TestAcidGuarantees/7c7f87a6124bfe2f4049ee7803e0f98c/A/4ca189b9a842483db6e475afcc8899aa 2024-11-16T09:58:57,105 DEBUG [StoreCloser-TestAcidGuarantees,,1731751108989.7c7f87a6124bfe2f4049ee7803e0f98c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/7c7f87a6124bfe2f4049ee7803e0f98c/A/64fcc2a7d41e439a952ea7e0969d6e41 to hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/archive/data/default/TestAcidGuarantees/7c7f87a6124bfe2f4049ee7803e0f98c/A/64fcc2a7d41e439a952ea7e0969d6e41 2024-11-16T09:58:57,106 DEBUG [StoreCloser-TestAcidGuarantees,,1731751108989.7c7f87a6124bfe2f4049ee7803e0f98c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/7c7f87a6124bfe2f4049ee7803e0f98c/A/8c843a2fb4074619b6de587720d98901 to hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/archive/data/default/TestAcidGuarantees/7c7f87a6124bfe2f4049ee7803e0f98c/A/8c843a2fb4074619b6de587720d98901 2024-11-16T09:58:57,107 DEBUG [StoreCloser-TestAcidGuarantees,,1731751108989.7c7f87a6124bfe2f4049ee7803e0f98c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/7c7f87a6124bfe2f4049ee7803e0f98c/A/2dddce9f3cd147d3b9cd2f2d801fc58a to hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/archive/data/default/TestAcidGuarantees/7c7f87a6124bfe2f4049ee7803e0f98c/A/2dddce9f3cd147d3b9cd2f2d801fc58a 2024-11-16T09:58:57,108 DEBUG [StoreCloser-TestAcidGuarantees,,1731751108989.7c7f87a6124bfe2f4049ee7803e0f98c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/7c7f87a6124bfe2f4049ee7803e0f98c/A/3bff852129eb44848826f920130a1069 to hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/archive/data/default/TestAcidGuarantees/7c7f87a6124bfe2f4049ee7803e0f98c/A/3bff852129eb44848826f920130a1069 2024-11-16T09:58:57,109 DEBUG [StoreCloser-TestAcidGuarantees,,1731751108989.7c7f87a6124bfe2f4049ee7803e0f98c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/7c7f87a6124bfe2f4049ee7803e0f98c/A/42e604f249434caf82bcc0dc3532e984 to hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/archive/data/default/TestAcidGuarantees/7c7f87a6124bfe2f4049ee7803e0f98c/A/42e604f249434caf82bcc0dc3532e984 2024-11-16T09:58:57,110 DEBUG [StoreCloser-TestAcidGuarantees,,1731751108989.7c7f87a6124bfe2f4049ee7803e0f98c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/7c7f87a6124bfe2f4049ee7803e0f98c/A/6b3a4c4820f0410ba008ad14ccf8af07 to hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/archive/data/default/TestAcidGuarantees/7c7f87a6124bfe2f4049ee7803e0f98c/A/6b3a4c4820f0410ba008ad14ccf8af07 2024-11-16T09:58:57,111 DEBUG [StoreCloser-TestAcidGuarantees,,1731751108989.7c7f87a6124bfe2f4049ee7803e0f98c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/7c7f87a6124bfe2f4049ee7803e0f98c/A/9db025a49e5b45598af87c7bf658d34b to hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/archive/data/default/TestAcidGuarantees/7c7f87a6124bfe2f4049ee7803e0f98c/A/9db025a49e5b45598af87c7bf658d34b 2024-11-16T09:58:57,112 DEBUG [StoreCloser-TestAcidGuarantees,,1731751108989.7c7f87a6124bfe2f4049ee7803e0f98c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/7c7f87a6124bfe2f4049ee7803e0f98c/A/2049d15304474f38af50e9ff363d240f to hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/archive/data/default/TestAcidGuarantees/7c7f87a6124bfe2f4049ee7803e0f98c/A/2049d15304474f38af50e9ff363d240f 2024-11-16T09:58:57,113 DEBUG [StoreCloser-TestAcidGuarantees,,1731751108989.7c7f87a6124bfe2f4049ee7803e0f98c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/7c7f87a6124bfe2f4049ee7803e0f98c/A/15c9a113751c448ab4ea0ce9821776a1 to hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/archive/data/default/TestAcidGuarantees/7c7f87a6124bfe2f4049ee7803e0f98c/A/15c9a113751c448ab4ea0ce9821776a1 2024-11-16T09:58:57,114 DEBUG [StoreCloser-TestAcidGuarantees,,1731751108989.7c7f87a6124bfe2f4049ee7803e0f98c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/7c7f87a6124bfe2f4049ee7803e0f98c/A/36f2297c7adc4ecaaf1ca484ad85f45f to hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/archive/data/default/TestAcidGuarantees/7c7f87a6124bfe2f4049ee7803e0f98c/A/36f2297c7adc4ecaaf1ca484ad85f45f 2024-11-16T09:58:57,115 DEBUG [StoreCloser-TestAcidGuarantees,,1731751108989.7c7f87a6124bfe2f4049ee7803e0f98c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/7c7f87a6124bfe2f4049ee7803e0f98c/A/cf911fd937c34d99a23ab70841f56bfe to hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/archive/data/default/TestAcidGuarantees/7c7f87a6124bfe2f4049ee7803e0f98c/A/cf911fd937c34d99a23ab70841f56bfe 2024-11-16T09:58:57,116 DEBUG [StoreCloser-TestAcidGuarantees,,1731751108989.7c7f87a6124bfe2f4049ee7803e0f98c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/7c7f87a6124bfe2f4049ee7803e0f98c/A/5ea7fb182fe64f0b9234c748529c1dec to hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/archive/data/default/TestAcidGuarantees/7c7f87a6124bfe2f4049ee7803e0f98c/A/5ea7fb182fe64f0b9234c748529c1dec 2024-11-16T09:58:57,117 DEBUG [StoreCloser-TestAcidGuarantees,,1731751108989.7c7f87a6124bfe2f4049ee7803e0f98c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/7c7f87a6124bfe2f4049ee7803e0f98c/A/34e9400bce784c0fa95be143ab8eefe9 to hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/archive/data/default/TestAcidGuarantees/7c7f87a6124bfe2f4049ee7803e0f98c/A/34e9400bce784c0fa95be143ab8eefe9 2024-11-16T09:58:57,118 DEBUG [StoreCloser-TestAcidGuarantees,,1731751108989.7c7f87a6124bfe2f4049ee7803e0f98c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/7c7f87a6124bfe2f4049ee7803e0f98c/A/4ffc1030f8104b6ba06470b62b7108f6 to hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/archive/data/default/TestAcidGuarantees/7c7f87a6124bfe2f4049ee7803e0f98c/A/4ffc1030f8104b6ba06470b62b7108f6 2024-11-16T09:58:57,119 DEBUG [StoreCloser-TestAcidGuarantees,,1731751108989.7c7f87a6124bfe2f4049ee7803e0f98c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/7c7f87a6124bfe2f4049ee7803e0f98c/A/76d20ead6b9f46678cadd75d8a84ae44 to hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/archive/data/default/TestAcidGuarantees/7c7f87a6124bfe2f4049ee7803e0f98c/A/76d20ead6b9f46678cadd75d8a84ae44 2024-11-16T09:58:57,120 DEBUG [StoreCloser-TestAcidGuarantees,,1731751108989.7c7f87a6124bfe2f4049ee7803e0f98c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/7c7f87a6124bfe2f4049ee7803e0f98c/A/d3490b0bbc89425ca64d2102a813fa1f to hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/archive/data/default/TestAcidGuarantees/7c7f87a6124bfe2f4049ee7803e0f98c/A/d3490b0bbc89425ca64d2102a813fa1f 2024-11-16T09:58:57,122 DEBUG [StoreCloser-TestAcidGuarantees,,1731751108989.7c7f87a6124bfe2f4049ee7803e0f98c.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/7c7f87a6124bfe2f4049ee7803e0f98c/B/bf8668dd1fd7427caec8099ddc5334f7, hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/7c7f87a6124bfe2f4049ee7803e0f98c/B/5e27822b25b0488bb4324b737dfd6709, hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/7c7f87a6124bfe2f4049ee7803e0f98c/B/1dd7fb59b8be4b04ae256d3db0407388, hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/7c7f87a6124bfe2f4049ee7803e0f98c/B/b471d622e7e145b989bbb16c6e434c3c, hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/7c7f87a6124bfe2f4049ee7803e0f98c/B/b4b46e5bd82a454d819133a11c7a80cb, hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/7c7f87a6124bfe2f4049ee7803e0f98c/B/85b3c54b164145d2925f3a8bb12ce8fe, hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/7c7f87a6124bfe2f4049ee7803e0f98c/B/61c3f1d80f724fefaa1bfbedddebbb01, hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/7c7f87a6124bfe2f4049ee7803e0f98c/B/84b81a5e733b45f88388e7fc5aeae30f, hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/7c7f87a6124bfe2f4049ee7803e0f98c/B/ed2ff962b6c54b2e95ec44bbec83832f, hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/7c7f87a6124bfe2f4049ee7803e0f98c/B/df3050c6104a4f5a843ce0d720c0bd8c, hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/7c7f87a6124bfe2f4049ee7803e0f98c/B/2bf5259150774f07a21de5b3fb39be24, hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/7c7f87a6124bfe2f4049ee7803e0f98c/B/cb196474d7ac4eaf876dc872b6eed2f4, hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/7c7f87a6124bfe2f4049ee7803e0f98c/B/bc151e55d37549828dba2a7feccb86eb, hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/7c7f87a6124bfe2f4049ee7803e0f98c/B/9989e7c7af994090af37ab514c81a8d1, hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/7c7f87a6124bfe2f4049ee7803e0f98c/B/0390cee6611b4f5b8512cca4042a155d, hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/7c7f87a6124bfe2f4049ee7803e0f98c/B/a59127e94a214b4db7e66865fc9a2d7a, hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/7c7f87a6124bfe2f4049ee7803e0f98c/B/a45924af8cf74348bbdc6ab01952e6e7, hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/7c7f87a6124bfe2f4049ee7803e0f98c/B/82d565f1aaa9448aaf40a4bdcf5f4ddc, hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/7c7f87a6124bfe2f4049ee7803e0f98c/B/cdd6bdee112a48518295ab24f2e5e365, hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/7c7f87a6124bfe2f4049ee7803e0f98c/B/d5f199b181be423483c2bb9c67b0a823, hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/7c7f87a6124bfe2f4049ee7803e0f98c/B/e21881a409cd409893e1543bb8f3f4d3, hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/7c7f87a6124bfe2f4049ee7803e0f98c/B/724e91f5d6c845aeaf175fa51a5f0e39, hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/7c7f87a6124bfe2f4049ee7803e0f98c/B/0f9b87aa526f4d468458a3048268b749, hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/7c7f87a6124bfe2f4049ee7803e0f98c/B/96fd5ab09a8e47cba98b0b1f2bcda7a3] to archive 2024-11-16T09:58:57,123 DEBUG [StoreCloser-TestAcidGuarantees,,1731751108989.7c7f87a6124bfe2f4049ee7803e0f98c.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-16T09:58:57,124 DEBUG [StoreCloser-TestAcidGuarantees,,1731751108989.7c7f87a6124bfe2f4049ee7803e0f98c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/7c7f87a6124bfe2f4049ee7803e0f98c/B/bf8668dd1fd7427caec8099ddc5334f7 to hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/archive/data/default/TestAcidGuarantees/7c7f87a6124bfe2f4049ee7803e0f98c/B/bf8668dd1fd7427caec8099ddc5334f7 2024-11-16T09:58:57,125 DEBUG [StoreCloser-TestAcidGuarantees,,1731751108989.7c7f87a6124bfe2f4049ee7803e0f98c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/7c7f87a6124bfe2f4049ee7803e0f98c/B/5e27822b25b0488bb4324b737dfd6709 to hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/archive/data/default/TestAcidGuarantees/7c7f87a6124bfe2f4049ee7803e0f98c/B/5e27822b25b0488bb4324b737dfd6709 2024-11-16T09:58:57,126 DEBUG [StoreCloser-TestAcidGuarantees,,1731751108989.7c7f87a6124bfe2f4049ee7803e0f98c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/7c7f87a6124bfe2f4049ee7803e0f98c/B/1dd7fb59b8be4b04ae256d3db0407388 to hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/archive/data/default/TestAcidGuarantees/7c7f87a6124bfe2f4049ee7803e0f98c/B/1dd7fb59b8be4b04ae256d3db0407388 2024-11-16T09:58:57,127 DEBUG [StoreCloser-TestAcidGuarantees,,1731751108989.7c7f87a6124bfe2f4049ee7803e0f98c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/7c7f87a6124bfe2f4049ee7803e0f98c/B/b471d622e7e145b989bbb16c6e434c3c to hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/archive/data/default/TestAcidGuarantees/7c7f87a6124bfe2f4049ee7803e0f98c/B/b471d622e7e145b989bbb16c6e434c3c 2024-11-16T09:58:57,128 DEBUG [StoreCloser-TestAcidGuarantees,,1731751108989.7c7f87a6124bfe2f4049ee7803e0f98c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/7c7f87a6124bfe2f4049ee7803e0f98c/B/b4b46e5bd82a454d819133a11c7a80cb to hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/archive/data/default/TestAcidGuarantees/7c7f87a6124bfe2f4049ee7803e0f98c/B/b4b46e5bd82a454d819133a11c7a80cb 2024-11-16T09:58:57,129 DEBUG [StoreCloser-TestAcidGuarantees,,1731751108989.7c7f87a6124bfe2f4049ee7803e0f98c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/7c7f87a6124bfe2f4049ee7803e0f98c/B/85b3c54b164145d2925f3a8bb12ce8fe to hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/archive/data/default/TestAcidGuarantees/7c7f87a6124bfe2f4049ee7803e0f98c/B/85b3c54b164145d2925f3a8bb12ce8fe 2024-11-16T09:58:57,130 DEBUG [StoreCloser-TestAcidGuarantees,,1731751108989.7c7f87a6124bfe2f4049ee7803e0f98c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/7c7f87a6124bfe2f4049ee7803e0f98c/B/61c3f1d80f724fefaa1bfbedddebbb01 to hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/archive/data/default/TestAcidGuarantees/7c7f87a6124bfe2f4049ee7803e0f98c/B/61c3f1d80f724fefaa1bfbedddebbb01 2024-11-16T09:58:57,131 DEBUG [StoreCloser-TestAcidGuarantees,,1731751108989.7c7f87a6124bfe2f4049ee7803e0f98c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/7c7f87a6124bfe2f4049ee7803e0f98c/B/84b81a5e733b45f88388e7fc5aeae30f to hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/archive/data/default/TestAcidGuarantees/7c7f87a6124bfe2f4049ee7803e0f98c/B/84b81a5e733b45f88388e7fc5aeae30f 2024-11-16T09:58:57,132 DEBUG [StoreCloser-TestAcidGuarantees,,1731751108989.7c7f87a6124bfe2f4049ee7803e0f98c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/7c7f87a6124bfe2f4049ee7803e0f98c/B/ed2ff962b6c54b2e95ec44bbec83832f to hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/archive/data/default/TestAcidGuarantees/7c7f87a6124bfe2f4049ee7803e0f98c/B/ed2ff962b6c54b2e95ec44bbec83832f 2024-11-16T09:58:57,133 DEBUG [StoreCloser-TestAcidGuarantees,,1731751108989.7c7f87a6124bfe2f4049ee7803e0f98c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/7c7f87a6124bfe2f4049ee7803e0f98c/B/df3050c6104a4f5a843ce0d720c0bd8c to hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/archive/data/default/TestAcidGuarantees/7c7f87a6124bfe2f4049ee7803e0f98c/B/df3050c6104a4f5a843ce0d720c0bd8c 2024-11-16T09:58:57,134 DEBUG [StoreCloser-TestAcidGuarantees,,1731751108989.7c7f87a6124bfe2f4049ee7803e0f98c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/7c7f87a6124bfe2f4049ee7803e0f98c/B/2bf5259150774f07a21de5b3fb39be24 to hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/archive/data/default/TestAcidGuarantees/7c7f87a6124bfe2f4049ee7803e0f98c/B/2bf5259150774f07a21de5b3fb39be24 2024-11-16T09:58:57,135 DEBUG [StoreCloser-TestAcidGuarantees,,1731751108989.7c7f87a6124bfe2f4049ee7803e0f98c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/7c7f87a6124bfe2f4049ee7803e0f98c/B/cb196474d7ac4eaf876dc872b6eed2f4 to hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/archive/data/default/TestAcidGuarantees/7c7f87a6124bfe2f4049ee7803e0f98c/B/cb196474d7ac4eaf876dc872b6eed2f4 2024-11-16T09:58:57,136 DEBUG [StoreCloser-TestAcidGuarantees,,1731751108989.7c7f87a6124bfe2f4049ee7803e0f98c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/7c7f87a6124bfe2f4049ee7803e0f98c/B/bc151e55d37549828dba2a7feccb86eb to hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/archive/data/default/TestAcidGuarantees/7c7f87a6124bfe2f4049ee7803e0f98c/B/bc151e55d37549828dba2a7feccb86eb 2024-11-16T09:58:57,137 DEBUG [StoreCloser-TestAcidGuarantees,,1731751108989.7c7f87a6124bfe2f4049ee7803e0f98c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/7c7f87a6124bfe2f4049ee7803e0f98c/B/9989e7c7af994090af37ab514c81a8d1 to hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/archive/data/default/TestAcidGuarantees/7c7f87a6124bfe2f4049ee7803e0f98c/B/9989e7c7af994090af37ab514c81a8d1 2024-11-16T09:58:57,138 DEBUG [StoreCloser-TestAcidGuarantees,,1731751108989.7c7f87a6124bfe2f4049ee7803e0f98c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/7c7f87a6124bfe2f4049ee7803e0f98c/B/0390cee6611b4f5b8512cca4042a155d to hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/archive/data/default/TestAcidGuarantees/7c7f87a6124bfe2f4049ee7803e0f98c/B/0390cee6611b4f5b8512cca4042a155d 2024-11-16T09:58:57,139 DEBUG [StoreCloser-TestAcidGuarantees,,1731751108989.7c7f87a6124bfe2f4049ee7803e0f98c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/7c7f87a6124bfe2f4049ee7803e0f98c/B/a59127e94a214b4db7e66865fc9a2d7a to hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/archive/data/default/TestAcidGuarantees/7c7f87a6124bfe2f4049ee7803e0f98c/B/a59127e94a214b4db7e66865fc9a2d7a 2024-11-16T09:58:57,140 DEBUG [StoreCloser-TestAcidGuarantees,,1731751108989.7c7f87a6124bfe2f4049ee7803e0f98c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/7c7f87a6124bfe2f4049ee7803e0f98c/B/a45924af8cf74348bbdc6ab01952e6e7 to hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/archive/data/default/TestAcidGuarantees/7c7f87a6124bfe2f4049ee7803e0f98c/B/a45924af8cf74348bbdc6ab01952e6e7 2024-11-16T09:58:57,141 DEBUG [StoreCloser-TestAcidGuarantees,,1731751108989.7c7f87a6124bfe2f4049ee7803e0f98c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/7c7f87a6124bfe2f4049ee7803e0f98c/B/82d565f1aaa9448aaf40a4bdcf5f4ddc to hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/archive/data/default/TestAcidGuarantees/7c7f87a6124bfe2f4049ee7803e0f98c/B/82d565f1aaa9448aaf40a4bdcf5f4ddc 2024-11-16T09:58:57,142 DEBUG [StoreCloser-TestAcidGuarantees,,1731751108989.7c7f87a6124bfe2f4049ee7803e0f98c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/7c7f87a6124bfe2f4049ee7803e0f98c/B/cdd6bdee112a48518295ab24f2e5e365 to hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/archive/data/default/TestAcidGuarantees/7c7f87a6124bfe2f4049ee7803e0f98c/B/cdd6bdee112a48518295ab24f2e5e365 2024-11-16T09:58:57,143 DEBUG [StoreCloser-TestAcidGuarantees,,1731751108989.7c7f87a6124bfe2f4049ee7803e0f98c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/7c7f87a6124bfe2f4049ee7803e0f98c/B/d5f199b181be423483c2bb9c67b0a823 to hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/archive/data/default/TestAcidGuarantees/7c7f87a6124bfe2f4049ee7803e0f98c/B/d5f199b181be423483c2bb9c67b0a823 2024-11-16T09:58:57,144 DEBUG [StoreCloser-TestAcidGuarantees,,1731751108989.7c7f87a6124bfe2f4049ee7803e0f98c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/7c7f87a6124bfe2f4049ee7803e0f98c/B/e21881a409cd409893e1543bb8f3f4d3 to hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/archive/data/default/TestAcidGuarantees/7c7f87a6124bfe2f4049ee7803e0f98c/B/e21881a409cd409893e1543bb8f3f4d3 2024-11-16T09:58:57,145 DEBUG [StoreCloser-TestAcidGuarantees,,1731751108989.7c7f87a6124bfe2f4049ee7803e0f98c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/7c7f87a6124bfe2f4049ee7803e0f98c/B/724e91f5d6c845aeaf175fa51a5f0e39 to hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/archive/data/default/TestAcidGuarantees/7c7f87a6124bfe2f4049ee7803e0f98c/B/724e91f5d6c845aeaf175fa51a5f0e39 2024-11-16T09:58:57,146 DEBUG [StoreCloser-TestAcidGuarantees,,1731751108989.7c7f87a6124bfe2f4049ee7803e0f98c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/7c7f87a6124bfe2f4049ee7803e0f98c/B/0f9b87aa526f4d468458a3048268b749 to hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/archive/data/default/TestAcidGuarantees/7c7f87a6124bfe2f4049ee7803e0f98c/B/0f9b87aa526f4d468458a3048268b749 2024-11-16T09:58:57,147 DEBUG [StoreCloser-TestAcidGuarantees,,1731751108989.7c7f87a6124bfe2f4049ee7803e0f98c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/7c7f87a6124bfe2f4049ee7803e0f98c/B/96fd5ab09a8e47cba98b0b1f2bcda7a3 to hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/archive/data/default/TestAcidGuarantees/7c7f87a6124bfe2f4049ee7803e0f98c/B/96fd5ab09a8e47cba98b0b1f2bcda7a3 2024-11-16T09:58:57,149 DEBUG [StoreCloser-TestAcidGuarantees,,1731751108989.7c7f87a6124bfe2f4049ee7803e0f98c.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/7c7f87a6124bfe2f4049ee7803e0f98c/C/ccfea16deb0641a3ab89085b418edb9b, hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/7c7f87a6124bfe2f4049ee7803e0f98c/C/95998a9414674875803abd21d398ade9, hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/7c7f87a6124bfe2f4049ee7803e0f98c/C/8ee49e7518574d3b98cb848b8d210202, hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/7c7f87a6124bfe2f4049ee7803e0f98c/C/39d84edc31624bc4bca34f4812ae659d, hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/7c7f87a6124bfe2f4049ee7803e0f98c/C/f7adaf3740b84fe18c83f162e49403a7, hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/7c7f87a6124bfe2f4049ee7803e0f98c/C/0b8fa5e40ba242e6a84abbb9e8e4fea5, hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/7c7f87a6124bfe2f4049ee7803e0f98c/C/821acc26a10c434aad651ccdb01cfe0a, hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/7c7f87a6124bfe2f4049ee7803e0f98c/C/2e3828d7d79b488a9c4d75f3a4e6af2d, hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/7c7f87a6124bfe2f4049ee7803e0f98c/C/3ec8bedcb3c04f9f96a9d4d23b6bd596, hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/7c7f87a6124bfe2f4049ee7803e0f98c/C/cd3257160203416e884bed4bb651ac18, hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/7c7f87a6124bfe2f4049ee7803e0f98c/C/20dd8e2f925044debf6455fa91c2494b, hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/7c7f87a6124bfe2f4049ee7803e0f98c/C/2f2c7dd9248d4963b68fdbffbc6a8106, hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/7c7f87a6124bfe2f4049ee7803e0f98c/C/43bc34e4ddad4cbc85fe7916dd7fde9a, hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/7c7f87a6124bfe2f4049ee7803e0f98c/C/4890337659474a46a61bf83aba890fb0, hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/7c7f87a6124bfe2f4049ee7803e0f98c/C/a0721c8cf1244143b39fcb3fb61616b6, hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/7c7f87a6124bfe2f4049ee7803e0f98c/C/e1471e6f23124679a598ae6112b8b618, hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/7c7f87a6124bfe2f4049ee7803e0f98c/C/3064c932d4fc4dc49cae309d5c3bdfbe, hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/7c7f87a6124bfe2f4049ee7803e0f98c/C/c9f68351e8204e01acd0f22f12b4cc64, hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/7c7f87a6124bfe2f4049ee7803e0f98c/C/492eaebe25d74d938f56f67c6caf697f, hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/7c7f87a6124bfe2f4049ee7803e0f98c/C/c8cbef2eeb3749d4b5fe5998814b7135, hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/7c7f87a6124bfe2f4049ee7803e0f98c/C/713ae3ec75aa419e8acd7ec473839cc8, hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/7c7f87a6124bfe2f4049ee7803e0f98c/C/9c7d3a5858cc46d2aeb57bf167db023f, hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/7c7f87a6124bfe2f4049ee7803e0f98c/C/9b4c5b55ab9a47fb97b7f974d882af88, hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/7c7f87a6124bfe2f4049ee7803e0f98c/C/745a570470de439d839401a55fbe1de1] to archive 2024-11-16T09:58:57,149 DEBUG [StoreCloser-TestAcidGuarantees,,1731751108989.7c7f87a6124bfe2f4049ee7803e0f98c.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-16T09:58:57,151 DEBUG [StoreCloser-TestAcidGuarantees,,1731751108989.7c7f87a6124bfe2f4049ee7803e0f98c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/7c7f87a6124bfe2f4049ee7803e0f98c/C/ccfea16deb0641a3ab89085b418edb9b to hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/archive/data/default/TestAcidGuarantees/7c7f87a6124bfe2f4049ee7803e0f98c/C/ccfea16deb0641a3ab89085b418edb9b 2024-11-16T09:58:57,152 DEBUG [StoreCloser-TestAcidGuarantees,,1731751108989.7c7f87a6124bfe2f4049ee7803e0f98c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/7c7f87a6124bfe2f4049ee7803e0f98c/C/95998a9414674875803abd21d398ade9 to hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/archive/data/default/TestAcidGuarantees/7c7f87a6124bfe2f4049ee7803e0f98c/C/95998a9414674875803abd21d398ade9 2024-11-16T09:58:57,153 DEBUG [StoreCloser-TestAcidGuarantees,,1731751108989.7c7f87a6124bfe2f4049ee7803e0f98c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/7c7f87a6124bfe2f4049ee7803e0f98c/C/8ee49e7518574d3b98cb848b8d210202 to hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/archive/data/default/TestAcidGuarantees/7c7f87a6124bfe2f4049ee7803e0f98c/C/8ee49e7518574d3b98cb848b8d210202 2024-11-16T09:58:57,154 DEBUG [StoreCloser-TestAcidGuarantees,,1731751108989.7c7f87a6124bfe2f4049ee7803e0f98c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/7c7f87a6124bfe2f4049ee7803e0f98c/C/39d84edc31624bc4bca34f4812ae659d to hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/archive/data/default/TestAcidGuarantees/7c7f87a6124bfe2f4049ee7803e0f98c/C/39d84edc31624bc4bca34f4812ae659d 2024-11-16T09:58:57,155 DEBUG [StoreCloser-TestAcidGuarantees,,1731751108989.7c7f87a6124bfe2f4049ee7803e0f98c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/7c7f87a6124bfe2f4049ee7803e0f98c/C/f7adaf3740b84fe18c83f162e49403a7 to hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/archive/data/default/TestAcidGuarantees/7c7f87a6124bfe2f4049ee7803e0f98c/C/f7adaf3740b84fe18c83f162e49403a7 2024-11-16T09:58:57,155 DEBUG [StoreCloser-TestAcidGuarantees,,1731751108989.7c7f87a6124bfe2f4049ee7803e0f98c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/7c7f87a6124bfe2f4049ee7803e0f98c/C/0b8fa5e40ba242e6a84abbb9e8e4fea5 to hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/archive/data/default/TestAcidGuarantees/7c7f87a6124bfe2f4049ee7803e0f98c/C/0b8fa5e40ba242e6a84abbb9e8e4fea5 2024-11-16T09:58:57,157 DEBUG [StoreCloser-TestAcidGuarantees,,1731751108989.7c7f87a6124bfe2f4049ee7803e0f98c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/7c7f87a6124bfe2f4049ee7803e0f98c/C/821acc26a10c434aad651ccdb01cfe0a to hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/archive/data/default/TestAcidGuarantees/7c7f87a6124bfe2f4049ee7803e0f98c/C/821acc26a10c434aad651ccdb01cfe0a 2024-11-16T09:58:57,158 DEBUG [StoreCloser-TestAcidGuarantees,,1731751108989.7c7f87a6124bfe2f4049ee7803e0f98c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/7c7f87a6124bfe2f4049ee7803e0f98c/C/2e3828d7d79b488a9c4d75f3a4e6af2d to hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/archive/data/default/TestAcidGuarantees/7c7f87a6124bfe2f4049ee7803e0f98c/C/2e3828d7d79b488a9c4d75f3a4e6af2d 2024-11-16T09:58:57,158 DEBUG [StoreCloser-TestAcidGuarantees,,1731751108989.7c7f87a6124bfe2f4049ee7803e0f98c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/7c7f87a6124bfe2f4049ee7803e0f98c/C/3ec8bedcb3c04f9f96a9d4d23b6bd596 to hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/archive/data/default/TestAcidGuarantees/7c7f87a6124bfe2f4049ee7803e0f98c/C/3ec8bedcb3c04f9f96a9d4d23b6bd596 2024-11-16T09:58:57,159 DEBUG [StoreCloser-TestAcidGuarantees,,1731751108989.7c7f87a6124bfe2f4049ee7803e0f98c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/7c7f87a6124bfe2f4049ee7803e0f98c/C/cd3257160203416e884bed4bb651ac18 to hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/archive/data/default/TestAcidGuarantees/7c7f87a6124bfe2f4049ee7803e0f98c/C/cd3257160203416e884bed4bb651ac18 2024-11-16T09:58:57,160 DEBUG [StoreCloser-TestAcidGuarantees,,1731751108989.7c7f87a6124bfe2f4049ee7803e0f98c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/7c7f87a6124bfe2f4049ee7803e0f98c/C/20dd8e2f925044debf6455fa91c2494b to hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/archive/data/default/TestAcidGuarantees/7c7f87a6124bfe2f4049ee7803e0f98c/C/20dd8e2f925044debf6455fa91c2494b 2024-11-16T09:58:57,161 DEBUG [StoreCloser-TestAcidGuarantees,,1731751108989.7c7f87a6124bfe2f4049ee7803e0f98c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/7c7f87a6124bfe2f4049ee7803e0f98c/C/2f2c7dd9248d4963b68fdbffbc6a8106 to hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/archive/data/default/TestAcidGuarantees/7c7f87a6124bfe2f4049ee7803e0f98c/C/2f2c7dd9248d4963b68fdbffbc6a8106 2024-11-16T09:58:57,162 DEBUG [StoreCloser-TestAcidGuarantees,,1731751108989.7c7f87a6124bfe2f4049ee7803e0f98c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/7c7f87a6124bfe2f4049ee7803e0f98c/C/43bc34e4ddad4cbc85fe7916dd7fde9a to hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/archive/data/default/TestAcidGuarantees/7c7f87a6124bfe2f4049ee7803e0f98c/C/43bc34e4ddad4cbc85fe7916dd7fde9a 2024-11-16T09:58:57,163 DEBUG [StoreCloser-TestAcidGuarantees,,1731751108989.7c7f87a6124bfe2f4049ee7803e0f98c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/7c7f87a6124bfe2f4049ee7803e0f98c/C/4890337659474a46a61bf83aba890fb0 to hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/archive/data/default/TestAcidGuarantees/7c7f87a6124bfe2f4049ee7803e0f98c/C/4890337659474a46a61bf83aba890fb0 2024-11-16T09:58:57,164 DEBUG [StoreCloser-TestAcidGuarantees,,1731751108989.7c7f87a6124bfe2f4049ee7803e0f98c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/7c7f87a6124bfe2f4049ee7803e0f98c/C/a0721c8cf1244143b39fcb3fb61616b6 to hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/archive/data/default/TestAcidGuarantees/7c7f87a6124bfe2f4049ee7803e0f98c/C/a0721c8cf1244143b39fcb3fb61616b6 2024-11-16T09:58:57,165 DEBUG [StoreCloser-TestAcidGuarantees,,1731751108989.7c7f87a6124bfe2f4049ee7803e0f98c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/7c7f87a6124bfe2f4049ee7803e0f98c/C/e1471e6f23124679a598ae6112b8b618 to hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/archive/data/default/TestAcidGuarantees/7c7f87a6124bfe2f4049ee7803e0f98c/C/e1471e6f23124679a598ae6112b8b618 2024-11-16T09:58:57,166 DEBUG [StoreCloser-TestAcidGuarantees,,1731751108989.7c7f87a6124bfe2f4049ee7803e0f98c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/7c7f87a6124bfe2f4049ee7803e0f98c/C/3064c932d4fc4dc49cae309d5c3bdfbe to hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/archive/data/default/TestAcidGuarantees/7c7f87a6124bfe2f4049ee7803e0f98c/C/3064c932d4fc4dc49cae309d5c3bdfbe 2024-11-16T09:58:57,167 DEBUG [StoreCloser-TestAcidGuarantees,,1731751108989.7c7f87a6124bfe2f4049ee7803e0f98c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/7c7f87a6124bfe2f4049ee7803e0f98c/C/c9f68351e8204e01acd0f22f12b4cc64 to hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/archive/data/default/TestAcidGuarantees/7c7f87a6124bfe2f4049ee7803e0f98c/C/c9f68351e8204e01acd0f22f12b4cc64 2024-11-16T09:58:57,168 DEBUG [StoreCloser-TestAcidGuarantees,,1731751108989.7c7f87a6124bfe2f4049ee7803e0f98c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/7c7f87a6124bfe2f4049ee7803e0f98c/C/492eaebe25d74d938f56f67c6caf697f to hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/archive/data/default/TestAcidGuarantees/7c7f87a6124bfe2f4049ee7803e0f98c/C/492eaebe25d74d938f56f67c6caf697f 2024-11-16T09:58:57,169 DEBUG [StoreCloser-TestAcidGuarantees,,1731751108989.7c7f87a6124bfe2f4049ee7803e0f98c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/7c7f87a6124bfe2f4049ee7803e0f98c/C/c8cbef2eeb3749d4b5fe5998814b7135 to hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/archive/data/default/TestAcidGuarantees/7c7f87a6124bfe2f4049ee7803e0f98c/C/c8cbef2eeb3749d4b5fe5998814b7135 2024-11-16T09:58:57,170 DEBUG [StoreCloser-TestAcidGuarantees,,1731751108989.7c7f87a6124bfe2f4049ee7803e0f98c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/7c7f87a6124bfe2f4049ee7803e0f98c/C/713ae3ec75aa419e8acd7ec473839cc8 to hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/archive/data/default/TestAcidGuarantees/7c7f87a6124bfe2f4049ee7803e0f98c/C/713ae3ec75aa419e8acd7ec473839cc8 2024-11-16T09:58:57,171 DEBUG [StoreCloser-TestAcidGuarantees,,1731751108989.7c7f87a6124bfe2f4049ee7803e0f98c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/7c7f87a6124bfe2f4049ee7803e0f98c/C/9c7d3a5858cc46d2aeb57bf167db023f to hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/archive/data/default/TestAcidGuarantees/7c7f87a6124bfe2f4049ee7803e0f98c/C/9c7d3a5858cc46d2aeb57bf167db023f 2024-11-16T09:58:57,172 DEBUG [StoreCloser-TestAcidGuarantees,,1731751108989.7c7f87a6124bfe2f4049ee7803e0f98c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/7c7f87a6124bfe2f4049ee7803e0f98c/C/9b4c5b55ab9a47fb97b7f974d882af88 to hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/archive/data/default/TestAcidGuarantees/7c7f87a6124bfe2f4049ee7803e0f98c/C/9b4c5b55ab9a47fb97b7f974d882af88 2024-11-16T09:58:57,173 DEBUG [StoreCloser-TestAcidGuarantees,,1731751108989.7c7f87a6124bfe2f4049ee7803e0f98c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/7c7f87a6124bfe2f4049ee7803e0f98c/C/745a570470de439d839401a55fbe1de1 to hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/archive/data/default/TestAcidGuarantees/7c7f87a6124bfe2f4049ee7803e0f98c/C/745a570470de439d839401a55fbe1de1 2024-11-16T09:58:57,177 DEBUG [RS_CLOSE_REGION-regionserver/037c58a19ddb:0-0 {event_type=M_RS_CLOSE_REGION, pid=55}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/7c7f87a6124bfe2f4049ee7803e0f98c/recovered.edits/381.seqid, newMaxSeqId=381, maxSeqId=4 2024-11-16T09:58:57,178 INFO [RS_CLOSE_REGION-regionserver/037c58a19ddb:0-0 {event_type=M_RS_CLOSE_REGION, pid=55}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1731751108989.7c7f87a6124bfe2f4049ee7803e0f98c. 2024-11-16T09:58:57,178 DEBUG [RS_CLOSE_REGION-regionserver/037c58a19ddb:0-0 {event_type=M_RS_CLOSE_REGION, pid=55}] regionserver.HRegion(1676): Region close journal for 7c7f87a6124bfe2f4049ee7803e0f98c: Waiting for close lock at 1731751135435Running coprocessor pre-close hooks at 1731751135435Disabling compacts and flushes for region at 1731751135435Disabling writes for close at 1731751135436 (+1 ms)Obtaining lock to block concurrent updates at 1731751135436Preparing flush snapshotting stores in 7c7f87a6124bfe2f4049ee7803e0f98c at 1731751135436Finished memstore snapshotting TestAcidGuarantees,,1731751108989.7c7f87a6124bfe2f4049ee7803e0f98c., syncing WAL and waiting on mvcc, flushsize=dataSize=34350, getHeapSize=90720, getOffHeapSize=0, getCellsCount=750 at 1731751135436Flushing stores of TestAcidGuarantees,,1731751108989.7c7f87a6124bfe2f4049ee7803e0f98c. at 1731751135436Flushing 7c7f87a6124bfe2f4049ee7803e0f98c/A: creating writer at 1731751135437 (+1 ms)Flushing mob file 7c7f87a6124bfe2f4049ee7803e0f98c/A: appending metadata at 1731751135442 (+5 ms)Flushing mob file 7c7f87a6124bfe2f4049ee7803e0f98c/A: closing flushed file at 1731751135442Flushing 7c7f87a6124bfe2f4049ee7803e0f98c/A: appending metadata at 1731751135851 (+409 ms)Flushing 7c7f87a6124bfe2f4049ee7803e0f98c/A: closing flushed file at 1731751135851Flushing 7c7f87a6124bfe2f4049ee7803e0f98c/B: creating writer at 1731751136260 (+409 ms)Flushing 7c7f87a6124bfe2f4049ee7803e0f98c/B: appending metadata at 1731751136262 (+2 ms)Flushing 7c7f87a6124bfe2f4049ee7803e0f98c/B: closing flushed file at 1731751136262Flushing 7c7f87a6124bfe2f4049ee7803e0f98c/C: creating writer at 1731751136671 (+409 ms)Flushing 7c7f87a6124bfe2f4049ee7803e0f98c/C: appending metadata at 1731751136673 (+2 ms)Flushing 7c7f87a6124bfe2f4049ee7803e0f98c/C: closing flushed file at 1731751136673Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@3abe9db: reopening flushed file at 1731751137082 (+409 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@2938806d: reopening flushed file at 1731751137086 (+4 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@3f8e7c32: reopening flushed file at 1731751137090 (+4 ms)Finished flush of dataSize ~33.54 KB/34350, heapSize ~88.59 KB/90720, currentSize=0 B/0 for 7c7f87a6124bfe2f4049ee7803e0f98c in 1658ms, sequenceid=378, compaction requested=true at 1731751137094 (+4 ms)Writing region close event to WAL at 1731751137174 (+80 ms)Running coprocessor post-close hooks at 1731751137178 (+4 ms)Closed at 1731751137178 2024-11-16T09:58:57,181 INFO [RS_CLOSE_REGION-regionserver/037c58a19ddb:0-0 {event_type=M_RS_CLOSE_REGION, pid=55}] handler.UnassignRegionHandler(157): Closed 7c7f87a6124bfe2f4049ee7803e0f98c 2024-11-16T09:58:57,181 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=54 updating hbase:meta row=7c7f87a6124bfe2f4049ee7803e0f98c, regionState=CLOSED 2024-11-16T09:58:57,182 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=55, ppid=54, state=RUNNABLE, hasLock=false; CloseRegionProcedure 7c7f87a6124bfe2f4049ee7803e0f98c, server=037c58a19ddb,37267,1731751077844 because future has completed 2024-11-16T09:58:57,184 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=55, resume processing ppid=54 2024-11-16T09:58:57,184 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=55, ppid=54, state=SUCCESS, hasLock=false; CloseRegionProcedure 7c7f87a6124bfe2f4049ee7803e0f98c, server=037c58a19ddb,37267,1731751077844 in 1.9000 sec 2024-11-16T09:58:57,186 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=54, resume processing ppid=53 2024-11-16T09:58:57,186 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=54, ppid=53, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=7c7f87a6124bfe2f4049ee7803e0f98c, UNASSIGN in 1.9030 sec 2024-11-16T09:58:57,188 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=53, resume processing ppid=52 2024-11-16T09:58:57,188 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=53, ppid=52, state=SUCCESS, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees in 1.9070 sec 2024-11-16T09:58:57,189 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731751137189"}]},"ts":"1731751137189"} 2024-11-16T09:58:57,190 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLED in hbase:meta 2024-11-16T09:58:57,190 INFO [PEWorker-5 {}] procedure.DisableTableProcedure(296): Set TestAcidGuarantees to state=DISABLED 2024-11-16T09:58:57,192 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=52, state=SUCCESS, hasLock=false; DisableTableProcedure table=TestAcidGuarantees in 1.9170 sec 2024-11-16T09:58:57,423 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37877 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=52 2024-11-16T09:58:57,423 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DISABLE, Table Name: default:TestAcidGuarantees completed 2024-11-16T09:58:57,424 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37877 {}] master.HMaster$5(2570): Client=jenkins//172.17.0.2 delete TestAcidGuarantees 2024-11-16T09:58:57,425 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37877 {}] procedure2.ProcedureExecutor(1139): Stored pid=56, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees 2024-11-16T09:58:57,426 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(103): Waiting for RIT for pid=56, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-16T09:58:57,426 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37877 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=56 2024-11-16T09:58:57,426 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(115): Deleting regions from filesystem for pid=56, state=RUNNABLE:DELETE_TABLE_CLEAR_FS_LAYOUT, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-16T09:58:57,429 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(131): ARCHIVING hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/7c7f87a6124bfe2f4049ee7803e0f98c 2024-11-16T09:58:57,431 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(159): Archiving [FileablePath, hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/7c7f87a6124bfe2f4049ee7803e0f98c/A, FileablePath, hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/7c7f87a6124bfe2f4049ee7803e0f98c/B, FileablePath, hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/7c7f87a6124bfe2f4049ee7803e0f98c/C, FileablePath, hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/7c7f87a6124bfe2f4049ee7803e0f98c/recovered.edits] 2024-11-16T09:58:57,434 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/7c7f87a6124bfe2f4049ee7803e0f98c/A/3a58c36cb1cb4b19a02cd633e4f31471 to hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/archive/data/default/TestAcidGuarantees/7c7f87a6124bfe2f4049ee7803e0f98c/A/3a58c36cb1cb4b19a02cd633e4f31471 2024-11-16T09:58:57,435 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/7c7f87a6124bfe2f4049ee7803e0f98c/A/4e45d1dfc3424857bed4a7303fd573eb to hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/archive/data/default/TestAcidGuarantees/7c7f87a6124bfe2f4049ee7803e0f98c/A/4e45d1dfc3424857bed4a7303fd573eb 2024-11-16T09:58:57,436 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/7c7f87a6124bfe2f4049ee7803e0f98c/A/b136583a9ba84d67a51c93bf74a272b9 to hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/archive/data/default/TestAcidGuarantees/7c7f87a6124bfe2f4049ee7803e0f98c/A/b136583a9ba84d67a51c93bf74a272b9 2024-11-16T09:58:57,438 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/7c7f87a6124bfe2f4049ee7803e0f98c/B/0ff4b870412d4c84b679d600e6df9d98 to hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/archive/data/default/TestAcidGuarantees/7c7f87a6124bfe2f4049ee7803e0f98c/B/0ff4b870412d4c84b679d600e6df9d98 2024-11-16T09:58:57,439 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/7c7f87a6124bfe2f4049ee7803e0f98c/B/5ea8a2c65fcb4e1abf1189f016dde782 to hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/archive/data/default/TestAcidGuarantees/7c7f87a6124bfe2f4049ee7803e0f98c/B/5ea8a2c65fcb4e1abf1189f016dde782 2024-11-16T09:58:57,440 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/7c7f87a6124bfe2f4049ee7803e0f98c/B/825d9c7704f0471f940babf25293601e to hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/archive/data/default/TestAcidGuarantees/7c7f87a6124bfe2f4049ee7803e0f98c/B/825d9c7704f0471f940babf25293601e 2024-11-16T09:58:57,442 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/7c7f87a6124bfe2f4049ee7803e0f98c/C/1992ed43352e4e4aba1fb223c6698799 to hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/archive/data/default/TestAcidGuarantees/7c7f87a6124bfe2f4049ee7803e0f98c/C/1992ed43352e4e4aba1fb223c6698799 2024-11-16T09:58:57,443 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/7c7f87a6124bfe2f4049ee7803e0f98c/C/3d73c76130644d799940328ba53420cd to hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/archive/data/default/TestAcidGuarantees/7c7f87a6124bfe2f4049ee7803e0f98c/C/3d73c76130644d799940328ba53420cd 2024-11-16T09:58:57,444 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/7c7f87a6124bfe2f4049ee7803e0f98c/C/f621f228b55141e4abbbe1ee9bd02957 to hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/archive/data/default/TestAcidGuarantees/7c7f87a6124bfe2f4049ee7803e0f98c/C/f621f228b55141e4abbbe1ee9bd02957 2024-11-16T09:58:57,446 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/7c7f87a6124bfe2f4049ee7803e0f98c/recovered.edits/381.seqid to hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/archive/data/default/TestAcidGuarantees/7c7f87a6124bfe2f4049ee7803e0f98c/recovered.edits/381.seqid 2024-11-16T09:58:57,446 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(610): Deleted hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/7c7f87a6124bfe2f4049ee7803e0f98c 2024-11-16T09:58:57,447 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(313): Archived TestAcidGuarantees regions 2024-11-16T09:58:57,447 DEBUG [PEWorker-3 {}] backup.HFileArchiver(131): ARCHIVING hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3 2024-11-16T09:58:57,448 DEBUG [PEWorker-3 {}] backup.HFileArchiver(159): Archiving [FileablePath, hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A] 2024-11-16T09:58:57,451 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241116037412d84b294a3bacdddbe69c227702_7c7f87a6124bfe2f4049ee7803e0f98c to hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241116037412d84b294a3bacdddbe69c227702_7c7f87a6124bfe2f4049ee7803e0f98c 2024-11-16T09:58:57,453 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411161d9574c60d754541b75a9ad2fe2e7796_7c7f87a6124bfe2f4049ee7803e0f98c to hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411161d9574c60d754541b75a9ad2fe2e7796_7c7f87a6124bfe2f4049ee7803e0f98c 2024-11-16T09:58:57,454 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111624023427d774423487434234fccac8f7_7c7f87a6124bfe2f4049ee7803e0f98c to hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111624023427d774423487434234fccac8f7_7c7f87a6124bfe2f4049ee7803e0f98c 2024-11-16T09:58:57,455 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241116393b75585278420e926a6eb898d59507_7c7f87a6124bfe2f4049ee7803e0f98c to hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241116393b75585278420e926a6eb898d59507_7c7f87a6124bfe2f4049ee7803e0f98c 2024-11-16T09:58:57,456 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111650618aa9f5c94860bb796babe0a66499_7c7f87a6124bfe2f4049ee7803e0f98c to hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111650618aa9f5c94860bb796babe0a66499_7c7f87a6124bfe2f4049ee7803e0f98c 2024-11-16T09:58:57,457 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411165a2d53f04455440790a250a3b092bca6_7c7f87a6124bfe2f4049ee7803e0f98c to hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411165a2d53f04455440790a250a3b092bca6_7c7f87a6124bfe2f4049ee7803e0f98c 2024-11-16T09:58:57,458 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411167774bab456c84fd3bb8f867e522b2ee6_7c7f87a6124bfe2f4049ee7803e0f98c to hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411167774bab456c84fd3bb8f867e522b2ee6_7c7f87a6124bfe2f4049ee7803e0f98c 2024-11-16T09:58:57,459 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111693d75b69d9444da09a3e69d0d73ffec7_7c7f87a6124bfe2f4049ee7803e0f98c to hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111693d75b69d9444da09a3e69d0d73ffec7_7c7f87a6124bfe2f4049ee7803e0f98c 2024-11-16T09:58:57,460 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411169f1b7c09e8fd45fbae6889443a7ede76_7c7f87a6124bfe2f4049ee7803e0f98c to hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411169f1b7c09e8fd45fbae6889443a7ede76_7c7f87a6124bfe2f4049ee7803e0f98c 2024-11-16T09:58:57,461 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241116a398894a58d24b12be2a9d8e28ed5e03_7c7f87a6124bfe2f4049ee7803e0f98c to hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241116a398894a58d24b12be2a9d8e28ed5e03_7c7f87a6124bfe2f4049ee7803e0f98c 2024-11-16T09:58:57,462 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241116a4bd5237f1004a00ae64e4d510126295_7c7f87a6124bfe2f4049ee7803e0f98c to hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241116a4bd5237f1004a00ae64e4d510126295_7c7f87a6124bfe2f4049ee7803e0f98c 2024-11-16T09:58:57,464 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241116c8fbfa4416f84c84bf534fe38bdd1430_7c7f87a6124bfe2f4049ee7803e0f98c to hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241116c8fbfa4416f84c84bf534fe38bdd1430_7c7f87a6124bfe2f4049ee7803e0f98c 2024-11-16T09:58:57,465 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241116da89ba3df1fe4b6eb99a016faaa5721d_7c7f87a6124bfe2f4049ee7803e0f98c to hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241116da89ba3df1fe4b6eb99a016faaa5721d_7c7f87a6124bfe2f4049ee7803e0f98c 2024-11-16T09:58:57,466 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241116dbe6e66079ca40bc8d84d8cc266ea18a_7c7f87a6124bfe2f4049ee7803e0f98c to hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241116dbe6e66079ca40bc8d84d8cc266ea18a_7c7f87a6124bfe2f4049ee7803e0f98c 2024-11-16T09:58:57,467 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241116e038e92c50cf48e597852abdb185a5c2_7c7f87a6124bfe2f4049ee7803e0f98c to hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241116e038e92c50cf48e597852abdb185a5c2_7c7f87a6124bfe2f4049ee7803e0f98c 2024-11-16T09:58:57,469 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241116e2f07f3fb6374ae8abf676c9d3395488_7c7f87a6124bfe2f4049ee7803e0f98c to hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241116e2f07f3fb6374ae8abf676c9d3395488_7c7f87a6124bfe2f4049ee7803e0f98c 2024-11-16T09:58:57,470 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241116e32f3ab2cc8743e18287927778b733e6_7c7f87a6124bfe2f4049ee7803e0f98c to hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241116e32f3ab2cc8743e18287927778b733e6_7c7f87a6124bfe2f4049ee7803e0f98c 2024-11-16T09:58:57,471 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241116e652a28518f24ba7b89f699d5f6a7615_7c7f87a6124bfe2f4049ee7803e0f98c to hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241116e652a28518f24ba7b89f699d5f6a7615_7c7f87a6124bfe2f4049ee7803e0f98c 2024-11-16T09:58:57,472 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241116f064654b48554817935f71a3f7529068_7c7f87a6124bfe2f4049ee7803e0f98c to hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241116f064654b48554817935f71a3f7529068_7c7f87a6124bfe2f4049ee7803e0f98c 2024-11-16T09:58:57,473 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241116f5691975dc39470781ff8c987c0076d4_7c7f87a6124bfe2f4049ee7803e0f98c to hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241116f5691975dc39470781ff8c987c0076d4_7c7f87a6124bfe2f4049ee7803e0f98c 2024-11-16T09:58:57,474 DEBUG [PEWorker-3 {}] backup.HFileArchiver(610): Deleted hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3 2024-11-16T09:58:57,476 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(120): Deleting regions from META for pid=56, state=RUNNABLE:DELETE_TABLE_REMOVE_FROM_META, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-16T09:58:57,479 WARN [PEWorker-3 {}] procedure.DeleteTableProcedure(370): Deleting some vestigial 1 rows of TestAcidGuarantees from hbase:meta 2024-11-16T09:58:57,481 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(407): Removing 'TestAcidGuarantees' descriptor. 2024-11-16T09:58:57,482 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(126): Deleting assignment state for pid=56, state=RUNNABLE:DELETE_TABLE_UNASSIGN_REGIONS, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-16T09:58:57,482 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(397): Removing 'TestAcidGuarantees' from region states. 2024-11-16T09:58:57,482 DEBUG [PEWorker-3 {}] assignment.RegionStateStore(723): Delete {"totalColumns":1,"row":"TestAcidGuarantees,,1731751108989.7c7f87a6124bfe2f4049ee7803e0f98c.","families":{"info":[{"qualifier":"","vlen":0,"tag":[],"timestamp":"1731751137482"}]},"ts":"9223372036854775807"} 2024-11-16T09:58:57,484 INFO [PEWorker-3 {}] assignment.RegionStateStore(562): Deleted 1 regions from META 2024-11-16T09:58:57,484 DEBUG [PEWorker-3 {}] assignment.RegionStateStore(563): Deleted regions: [{ENCODED => 7c7f87a6124bfe2f4049ee7803e0f98c, NAME => 'TestAcidGuarantees,,1731751108989.7c7f87a6124bfe2f4049ee7803e0f98c.', STARTKEY => '', ENDKEY => ''}] 2024-11-16T09:58:57,484 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(401): Marking 'TestAcidGuarantees' as deleted. 2024-11-16T09:58:57,484 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Delete {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":0,"tag":[],"timestamp":"1731751137484"}]},"ts":"9223372036854775807"} 2024-11-16T09:58:57,486 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(867): Deleted table TestAcidGuarantees state from META 2024-11-16T09:58:57,486 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(133): Finished pid=56, state=RUNNABLE:DELETE_TABLE_POST_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-16T09:58:57,487 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=56, state=SUCCESS, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees in 62 msec 2024-11-16T09:58:57,533 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37877 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=56 2024-11-16T09:58:57,533 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocator(219): Clear meta cache for TestAcidGuarantees 2024-11-16T09:58:57,533 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DELETE, Table Name: default:TestAcidGuarantees completed 2024-11-16T09:58:57,539 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(153): Removing adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_TestAcidGuarantees 2024-11-16T09:58:57,542 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: TestAcidGuaranteesWithNoInMemCompaction#testMobMixedAtomicity Thread=243 (was 241) Potentially hanging thread: Async disk worker #0 for volume /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/62476c2b-1e2d-7029-5a2f-94eb6a0fedba/cluster_7e615273-6ed6-1343-5769-ac1955575909/data/data2 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ProcedureExecutor-Async-Task-Executor-7 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: HFileArchiver-2 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: Async disk worker #0 for volume /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/62476c2b-1e2d-7029-5a2f-94eb6a0fedba/cluster_7e615273-6ed6-1343-5769-ac1955575909/data/data1 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-791136307_22 at /127.0.0.1:46922 [Waiting for operation #358] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ProcedureExecutor-Async-Task-Executor-6 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-1188164596_22 at /127.0.0.1:47126 [Waiting for operation #229] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-791136307_22 at /127.0.0.1:47106 [Waiting for operation #307] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-1188164596_22 at /127.0.0.1:47098 [Waiting for operation #333] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ProcedureExecutor-Async-Task-Executor-4 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-1188164596_22 at /127.0.0.1:39830 [Waiting for operation #779] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ProcedureExecutor-Async-Task-Executor-5 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=463 (was 463), MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=357 (was 309) - SystemLoadAverage LEAK? -, ProcessCount=11 (was 11), AvailableMemoryMB=6047 (was 5349) - AvailableMemoryMB LEAK? - 2024-11-16T09:58:57,552 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: TestAcidGuaranteesWithNoInMemCompaction#testGetAtomicity Thread=243, OpenFileDescriptor=463, MaxFileDescriptor=1048576, SystemLoadAverage=357, ProcessCount=11, AvailableMemoryMB=6047 2024-11-16T09:58:57,553 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37877 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (131072) is too small, which might cause very frequent flushing. 2024-11-16T09:58:57,553 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37877 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.2 create 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'NONE'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-16T09:58:57,554 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37877 {}] procedure2.ProcedureExecutor(1139): Stored pid=57, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=TestAcidGuarantees 2024-11-16T09:58:57,555 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=57, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_PRE_OPERATION 2024-11-16T09:58:57,555 DEBUG [PEWorker-2 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T09:58:57,555 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37877 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.2 procedure request for creating table: namespace: "default" qualifier: "TestAcidGuarantees" procId is: 57 2024-11-16T09:58:57,556 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37877 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=57 2024-11-16T09:58:57,556 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=57, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-11-16T09:58:57,561 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37591 is added to blk_1073742077_1253 (size=959) 2024-11-16T09:58:57,663 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37877 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=57 2024-11-16T09:58:57,872 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37877 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=57 2024-11-16T09:58:57,963 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => f3ed23ac403772e0fe4ca9ee892544ec, NAME => 'TestAcidGuarantees,,1731751137553.f3ed23ac403772e0fe4ca9ee892544ec.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'NONE', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592 2024-11-16T09:58:57,969 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37591 is added to blk_1073742078_1254 (size=53) 2024-11-16T09:58:58,183 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37877 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=57 2024-11-16T09:58:58,370 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1731751137553.f3ed23ac403772e0fe4ca9ee892544ec.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-16T09:58:58,370 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1722): Closing f3ed23ac403772e0fe4ca9ee892544ec, disabling compactions & flushes 2024-11-16T09:58:58,370 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1731751137553.f3ed23ac403772e0fe4ca9ee892544ec. 2024-11-16T09:58:58,370 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1731751137553.f3ed23ac403772e0fe4ca9ee892544ec. 2024-11-16T09:58:58,370 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1731751137553.f3ed23ac403772e0fe4ca9ee892544ec. after waiting 0 ms 2024-11-16T09:58:58,370 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1731751137553.f3ed23ac403772e0fe4ca9ee892544ec. 2024-11-16T09:58:58,370 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1731751137553.f3ed23ac403772e0fe4ca9ee892544ec. 2024-11-16T09:58:58,370 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1676): Region close journal for f3ed23ac403772e0fe4ca9ee892544ec: Waiting for close lock at 1731751138370Disabling compacts and flushes for region at 1731751138370Disabling writes for close at 1731751138370Writing region close event to WAL at 1731751138370Closed at 1731751138370 2024-11-16T09:58:58,371 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=57, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ADD_TO_META 2024-11-16T09:58:58,371 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"TestAcidGuarantees,,1731751137553.f3ed23ac403772e0fe4ca9ee892544ec.","families":{"info":[{"qualifier":"regioninfo","vlen":52,"tag":[],"timestamp":"1731751138371"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1731751138371"}]},"ts":"1731751138371"} 2024-11-16T09:58:58,373 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-11-16T09:58:58,373 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=57, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-11-16T09:58:58,374 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731751138373"}]},"ts":"1731751138373"} 2024-11-16T09:58:58,375 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLING in hbase:meta 2024-11-16T09:58:58,375 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=58, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=f3ed23ac403772e0fe4ca9ee892544ec, ASSIGN}] 2024-11-16T09:58:58,376 INFO [PEWorker-4 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=58, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=f3ed23ac403772e0fe4ca9ee892544ec, ASSIGN 2024-11-16T09:58:58,377 INFO [PEWorker-4 {}] assignment.TransitRegionStateProcedure(269): Starting pid=58, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=f3ed23ac403772e0fe4ca9ee892544ec, ASSIGN; state=OFFLINE, location=037c58a19ddb,37267,1731751077844; forceNewPlan=false, retain=false 2024-11-16T09:58:58,527 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=58 updating hbase:meta row=f3ed23ac403772e0fe4ca9ee892544ec, regionState=OPENING, regionLocation=037c58a19ddb,37267,1731751077844 2024-11-16T09:58:58,529 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=58, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=f3ed23ac403772e0fe4ca9ee892544ec, ASSIGN because future has completed 2024-11-16T09:58:58,529 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=59, ppid=58, state=RUNNABLE, hasLock=false; OpenRegionProcedure f3ed23ac403772e0fe4ca9ee892544ec, server=037c58a19ddb,37267,1731751077844}] 2024-11-16T09:58:58,683 INFO [RS_OPEN_REGION-regionserver/037c58a19ddb:0-0 {event_type=M_RS_OPEN_REGION, pid=59}] handler.AssignRegionHandler(132): Open TestAcidGuarantees,,1731751137553.f3ed23ac403772e0fe4ca9ee892544ec. 2024-11-16T09:58:58,683 DEBUG [RS_OPEN_REGION-regionserver/037c58a19ddb:0-0 {event_type=M_RS_OPEN_REGION, pid=59}] regionserver.HRegion(7752): Opening region: {ENCODED => f3ed23ac403772e0fe4ca9ee892544ec, NAME => 'TestAcidGuarantees,,1731751137553.f3ed23ac403772e0fe4ca9ee892544ec.', STARTKEY => '', ENDKEY => ''} 2024-11-16T09:58:58,683 DEBUG [RS_OPEN_REGION-regionserver/037c58a19ddb:0-0 {event_type=M_RS_OPEN_REGION, pid=59}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestAcidGuarantees f3ed23ac403772e0fe4ca9ee892544ec 2024-11-16T09:58:58,683 DEBUG [RS_OPEN_REGION-regionserver/037c58a19ddb:0-0 {event_type=M_RS_OPEN_REGION, pid=59}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1731751137553.f3ed23ac403772e0fe4ca9ee892544ec.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-16T09:58:58,683 DEBUG [RS_OPEN_REGION-regionserver/037c58a19ddb:0-0 {event_type=M_RS_OPEN_REGION, pid=59}] regionserver.HRegion(7794): checking encryption for f3ed23ac403772e0fe4ca9ee892544ec 2024-11-16T09:58:58,683 DEBUG [RS_OPEN_REGION-regionserver/037c58a19ddb:0-0 {event_type=M_RS_OPEN_REGION, pid=59}] regionserver.HRegion(7797): checking classloading for f3ed23ac403772e0fe4ca9ee892544ec 2024-11-16T09:58:58,685 INFO [StoreOpener-f3ed23ac403772e0fe4ca9ee892544ec-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family A of region f3ed23ac403772e0fe4ca9ee892544ec 2024-11-16T09:58:58,686 INFO [StoreOpener-f3ed23ac403772e0fe4ca9ee892544ec-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region f3ed23ac403772e0fe4ca9ee892544ec columnFamilyName A 2024-11-16T09:58:58,686 DEBUG [StoreOpener-f3ed23ac403772e0fe4ca9ee892544ec-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T09:58:58,686 INFO [StoreOpener-f3ed23ac403772e0fe4ca9ee892544ec-1 {}] regionserver.HStore(327): Store=f3ed23ac403772e0fe4ca9ee892544ec/A, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-16T09:58:58,686 INFO [StoreOpener-f3ed23ac403772e0fe4ca9ee892544ec-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family B of region f3ed23ac403772e0fe4ca9ee892544ec 2024-11-16T09:58:58,687 INFO [StoreOpener-f3ed23ac403772e0fe4ca9ee892544ec-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region f3ed23ac403772e0fe4ca9ee892544ec columnFamilyName B 2024-11-16T09:58:58,688 DEBUG [StoreOpener-f3ed23ac403772e0fe4ca9ee892544ec-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T09:58:58,688 INFO [StoreOpener-f3ed23ac403772e0fe4ca9ee892544ec-1 {}] regionserver.HStore(327): Store=f3ed23ac403772e0fe4ca9ee892544ec/B, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-16T09:58:58,688 INFO [StoreOpener-f3ed23ac403772e0fe4ca9ee892544ec-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family C of region f3ed23ac403772e0fe4ca9ee892544ec 2024-11-16T09:58:58,689 INFO [StoreOpener-f3ed23ac403772e0fe4ca9ee892544ec-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region f3ed23ac403772e0fe4ca9ee892544ec columnFamilyName C 2024-11-16T09:58:58,689 DEBUG [StoreOpener-f3ed23ac403772e0fe4ca9ee892544ec-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T09:58:58,689 INFO [StoreOpener-f3ed23ac403772e0fe4ca9ee892544ec-1 {}] regionserver.HStore(327): Store=f3ed23ac403772e0fe4ca9ee892544ec/C, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-16T09:58:58,689 DEBUG [RS_OPEN_REGION-regionserver/037c58a19ddb:0-0 {event_type=M_RS_OPEN_REGION, pid=59}] regionserver.HRegion(1038): replaying wal for f3ed23ac403772e0fe4ca9ee892544ec 2024-11-16T09:58:58,690 DEBUG [RS_OPEN_REGION-regionserver/037c58a19ddb:0-0 {event_type=M_RS_OPEN_REGION, pid=59}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/f3ed23ac403772e0fe4ca9ee892544ec 2024-11-16T09:58:58,690 DEBUG [RS_OPEN_REGION-regionserver/037c58a19ddb:0-0 {event_type=M_RS_OPEN_REGION, pid=59}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/f3ed23ac403772e0fe4ca9ee892544ec 2024-11-16T09:58:58,691 DEBUG [RS_OPEN_REGION-regionserver/037c58a19ddb:0-0 {event_type=M_RS_OPEN_REGION, pid=59}] regionserver.HRegion(1048): stopping wal replay for f3ed23ac403772e0fe4ca9ee892544ec 2024-11-16T09:58:58,691 DEBUG [RS_OPEN_REGION-regionserver/037c58a19ddb:0-0 {event_type=M_RS_OPEN_REGION, pid=59}] regionserver.HRegion(1060): Cleaning up temporary data for f3ed23ac403772e0fe4ca9ee892544ec 2024-11-16T09:58:58,691 DEBUG [RS_OPEN_REGION-regionserver/037c58a19ddb:0-0 {event_type=M_RS_OPEN_REGION, pid=59}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table TestAcidGuarantees descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-11-16T09:58:58,692 DEBUG [RS_OPEN_REGION-regionserver/037c58a19ddb:0-0 {event_type=M_RS_OPEN_REGION, pid=59}] regionserver.HRegion(1093): writing seq id for f3ed23ac403772e0fe4ca9ee892544ec 2024-11-16T09:58:58,693 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37877 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=57 2024-11-16T09:58:58,694 DEBUG [RS_OPEN_REGION-regionserver/037c58a19ddb:0-0 {event_type=M_RS_OPEN_REGION, pid=59}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/f3ed23ac403772e0fe4ca9ee892544ec/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-16T09:58:58,694 INFO [RS_OPEN_REGION-regionserver/037c58a19ddb:0-0 {event_type=M_RS_OPEN_REGION, pid=59}] regionserver.HRegion(1114): Opened f3ed23ac403772e0fe4ca9ee892544ec; next sequenceid=2; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=72544882, jitterRate=0.08100298047065735}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-11-16T09:58:58,694 DEBUG [RS_OPEN_REGION-regionserver/037c58a19ddb:0-0 {event_type=M_RS_OPEN_REGION, pid=59}] regionserver.HRegion(1122): Running coprocessor post-open hooks for f3ed23ac403772e0fe4ca9ee892544ec 2024-11-16T09:58:58,695 DEBUG [RS_OPEN_REGION-regionserver/037c58a19ddb:0-0 {event_type=M_RS_OPEN_REGION, pid=59}] regionserver.HRegion(1006): Region open journal for f3ed23ac403772e0fe4ca9ee892544ec: Running coprocessor pre-open hook at 1731751138683Writing region info on filesystem at 1731751138684 (+1 ms)Initializing all the Stores at 1731751138684Instantiating store for column family {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731751138684Instantiating store for column family {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731751138684Instantiating store for column family {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731751138684Cleaning up temporary data from old regions at 1731751138691 (+7 ms)Running coprocessor post-open hooks at 1731751138694 (+3 ms)Region opened successfully at 1731751138695 (+1 ms) 2024-11-16T09:58:58,696 INFO [RS_OPEN_REGION-regionserver/037c58a19ddb:0-0 {event_type=M_RS_OPEN_REGION, pid=59}] regionserver.HRegionServer(2236): Post open deploy tasks for TestAcidGuarantees,,1731751137553.f3ed23ac403772e0fe4ca9ee892544ec., pid=59, masterSystemTime=1731751138680 2024-11-16T09:58:58,698 DEBUG [RS_OPEN_REGION-regionserver/037c58a19ddb:0-0 {event_type=M_RS_OPEN_REGION, pid=59}] regionserver.HRegionServer(2266): Finished post open deploy task for TestAcidGuarantees,,1731751137553.f3ed23ac403772e0fe4ca9ee892544ec. 2024-11-16T09:58:58,698 INFO [RS_OPEN_REGION-regionserver/037c58a19ddb:0-0 {event_type=M_RS_OPEN_REGION, pid=59}] handler.AssignRegionHandler(153): Opened TestAcidGuarantees,,1731751137553.f3ed23ac403772e0fe4ca9ee892544ec. 2024-11-16T09:58:58,698 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=58 updating hbase:meta row=f3ed23ac403772e0fe4ca9ee892544ec, regionState=OPEN, openSeqNum=2, regionLocation=037c58a19ddb,37267,1731751077844 2024-11-16T09:58:58,699 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=59, ppid=58, state=RUNNABLE, hasLock=false; OpenRegionProcedure f3ed23ac403772e0fe4ca9ee892544ec, server=037c58a19ddb,37267,1731751077844 because future has completed 2024-11-16T09:58:58,701 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=59, resume processing ppid=58 2024-11-16T09:58:58,701 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=59, ppid=58, state=SUCCESS, hasLock=false; OpenRegionProcedure f3ed23ac403772e0fe4ca9ee892544ec, server=037c58a19ddb,37267,1731751077844 in 171 msec 2024-11-16T09:58:58,703 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=58, resume processing ppid=57 2024-11-16T09:58:58,703 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=58, ppid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=f3ed23ac403772e0fe4ca9ee892544ec, ASSIGN in 326 msec 2024-11-16T09:58:58,703 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=57, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-11-16T09:58:58,703 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731751138703"}]},"ts":"1731751138703"} 2024-11-16T09:58:58,705 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLED in hbase:meta 2024-11-16T09:58:58,705 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=57, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_POST_OPERATION 2024-11-16T09:58:58,707 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=57, state=SUCCESS, hasLock=false; CreateTableProcedure table=TestAcidGuarantees in 1.1520 sec 2024-11-16T09:58:59,702 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37877 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=57 2024-11-16T09:58:59,703 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: default:TestAcidGuarantees completed 2024-11-16T09:58:59,704 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@21c1be0, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T09:58:59,704 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 037c58a19ddb,37877,-1 for getting cluster id 2024-11-16T09:58:59,704 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-16T09:58:59,705 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '1fe46ee8-cb44-44b1-8251-37103629f786' 2024-11-16T09:58:59,705 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-16T09:58:59,705 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "1fe46ee8-cb44-44b1-8251-37103629f786" 2024-11-16T09:58:59,706 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4786aeab, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T09:58:59,706 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [037c58a19ddb,37877,-1] 2024-11-16T09:58:59,706 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-16T09:58:59,706 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T09:58:59,707 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:36634, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-16T09:58:59,707 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@73ffeec1, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T09:58:59,707 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-16T09:58:59,708 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=037c58a19ddb,37267,1731751077844, seqNum=-1] 2024-11-16T09:58:59,708 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-16T09:58:59,709 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:46636, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-16T09:58:59,710 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-11-16T09:58:59,711 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncConnectionImpl(321): The fetched master address is 037c58a19ddb,37877,1731751077131 2024-11-16T09:58:59,711 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@4977f21f 2024-11-16T09:58:59,711 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-11-16T09:58:59,712 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:36638, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-11-16T09:58:59,713 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1240be3c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T09:58:59,713 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 037c58a19ddb,37877,-1 for getting cluster id 2024-11-16T09:58:59,713 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-16T09:58:59,714 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '1fe46ee8-cb44-44b1-8251-37103629f786' 2024-11-16T09:58:59,714 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-16T09:58:59,714 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "1fe46ee8-cb44-44b1-8251-37103629f786" 2024-11-16T09:58:59,714 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4a3d6e97, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T09:58:59,714 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [037c58a19ddb,37877,-1] 2024-11-16T09:58:59,714 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-16T09:58:59,714 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T09:58:59,715 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:37942, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-16T09:58:59,715 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2f9eed95, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T09:58:59,716 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2885eb44, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T09:58:59,716 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 037c58a19ddb,37877,-1 for getting cluster id 2024-11-16T09:58:59,716 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-16T09:58:59,716 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '1fe46ee8-cb44-44b1-8251-37103629f786' 2024-11-16T09:58:59,716 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-16T09:58:59,716 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "1fe46ee8-cb44-44b1-8251-37103629f786" 2024-11-16T09:58:59,717 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3bb5923e, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T09:58:59,717 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [037c58a19ddb,37877,-1] 2024-11-16T09:58:59,717 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-16T09:58:59,717 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T09:58:59,717 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:37966, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-16T09:58:59,718 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@46824d0f, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T09:58:59,718 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@692a6227, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T09:58:59,718 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 037c58a19ddb,37877,-1 for getting cluster id 2024-11-16T09:58:59,719 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-16T09:58:59,719 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '1fe46ee8-cb44-44b1-8251-37103629f786' 2024-11-16T09:58:59,719 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-16T09:58:59,719 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "1fe46ee8-cb44-44b1-8251-37103629f786" 2024-11-16T09:58:59,719 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7f45b9f3, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T09:58:59,719 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [037c58a19ddb,37877,-1] 2024-11-16T09:58:59,719 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-16T09:58:59,720 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T09:58:59,720 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:37990, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-16T09:58:59,721 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2153b6f9, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T09:58:59,721 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@128d97bc, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T09:58:59,721 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 037c58a19ddb,37877,-1 for getting cluster id 2024-11-16T09:58:59,721 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-16T09:58:59,722 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '1fe46ee8-cb44-44b1-8251-37103629f786' 2024-11-16T09:58:59,722 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-16T09:58:59,722 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "1fe46ee8-cb44-44b1-8251-37103629f786" 2024-11-16T09:58:59,722 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6d16e55c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T09:58:59,722 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [037c58a19ddb,37877,-1] 2024-11-16T09:58:59,722 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-16T09:58:59,722 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T09:58:59,723 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:37998, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-16T09:58:59,724 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@57be07f7, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T09:58:59,724 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7acbee0d, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T09:58:59,724 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 037c58a19ddb,37877,-1 for getting cluster id 2024-11-16T09:58:59,724 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-16T09:58:59,725 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '1fe46ee8-cb44-44b1-8251-37103629f786' 2024-11-16T09:58:59,725 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-16T09:58:59,725 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "1fe46ee8-cb44-44b1-8251-37103629f786" 2024-11-16T09:58:59,725 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4365e966, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T09:58:59,725 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [037c58a19ddb,37877,-1] 2024-11-16T09:58:59,725 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-16T09:58:59,726 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T09:58:59,726 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:38018, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-16T09:58:59,727 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7ce0657a, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T09:58:59,727 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@47a5c877, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T09:58:59,727 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 037c58a19ddb,37877,-1 for getting cluster id 2024-11-16T09:58:59,727 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-16T09:58:59,728 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '1fe46ee8-cb44-44b1-8251-37103629f786' 2024-11-16T09:58:59,728 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-16T09:58:59,728 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "1fe46ee8-cb44-44b1-8251-37103629f786" 2024-11-16T09:58:59,728 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@68cbc33d, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T09:58:59,728 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [037c58a19ddb,37877,-1] 2024-11-16T09:58:59,728 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-16T09:58:59,728 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T09:58:59,729 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:38032, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-16T09:58:59,729 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@283affdc, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T09:58:59,730 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@679f7426, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T09:58:59,730 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 037c58a19ddb,37877,-1 for getting cluster id 2024-11-16T09:58:59,730 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-16T09:58:59,731 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '1fe46ee8-cb44-44b1-8251-37103629f786' 2024-11-16T09:58:59,731 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-16T09:58:59,731 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "1fe46ee8-cb44-44b1-8251-37103629f786" 2024-11-16T09:58:59,731 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3e183e2d, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T09:58:59,731 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [037c58a19ddb,37877,-1] 2024-11-16T09:58:59,731 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-16T09:58:59,731 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T09:58:59,732 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:38038, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-16T09:58:59,732 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1bfbfbea, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T09:58:59,733 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@b71af22, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T09:58:59,733 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 037c58a19ddb,37877,-1 for getting cluster id 2024-11-16T09:58:59,733 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-16T09:58:59,733 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '1fe46ee8-cb44-44b1-8251-37103629f786' 2024-11-16T09:58:59,733 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-16T09:58:59,733 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "1fe46ee8-cb44-44b1-8251-37103629f786" 2024-11-16T09:58:59,734 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5ac01ac4, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T09:58:59,734 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [037c58a19ddb,37877,-1] 2024-11-16T09:58:59,734 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-16T09:58:59,734 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T09:58:59,735 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:38056, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-16T09:58:59,735 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@645dcba, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T09:58:59,736 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@728d338, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T09:58:59,736 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 037c58a19ddb,37877,-1 for getting cluster id 2024-11-16T09:58:59,736 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-16T09:58:59,736 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '1fe46ee8-cb44-44b1-8251-37103629f786' 2024-11-16T09:58:59,736 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-16T09:58:59,736 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "1fe46ee8-cb44-44b1-8251-37103629f786" 2024-11-16T09:58:59,736 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3eda3267, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T09:58:59,736 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [037c58a19ddb,37877,-1] 2024-11-16T09:58:59,737 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-16T09:58:59,737 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T09:58:59,737 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:38084, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-16T09:58:59,738 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@735d7bf9, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T09:58:59,738 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@cc00bc6, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T09:58:59,738 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 037c58a19ddb,37877,-1 for getting cluster id 2024-11-16T09:58:59,738 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-16T09:58:59,739 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '1fe46ee8-cb44-44b1-8251-37103629f786' 2024-11-16T09:58:59,739 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-16T09:58:59,739 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "1fe46ee8-cb44-44b1-8251-37103629f786" 2024-11-16T09:58:59,739 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@57ef209d, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T09:58:59,739 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [037c58a19ddb,37877,-1] 2024-11-16T09:58:59,739 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-16T09:58:59,739 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T09:58:59,740 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:38094, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-16T09:58:59,740 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@631e98e9, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T09:58:59,741 DEBUG [Thread-1179 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-16T09:58:59,741 DEBUG [Thread-1188 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-16T09:58:59,741 DEBUG [Thread-1192 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-16T09:58:59,741 DEBUG [Thread-1194 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-16T09:58:59,741 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37877 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-16T09:58:59,741 DEBUG [Thread-1177 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-16T09:58:59,741 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=037c58a19ddb,37267,1731751077844, seqNum=-1] 2024-11-16T09:58:59,741 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=037c58a19ddb,37267,1731751077844, seqNum=-1] 2024-11-16T09:58:59,742 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=037c58a19ddb,37267,1731751077844, seqNum=-1] 2024-11-16T09:58:59,742 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-16T09:58:59,742 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-16T09:58:59,742 DEBUG [Thread-1183 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-16T09:58:59,742 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-16T09:58:59,742 DEBUG [Thread-1196 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-16T09:58:59,742 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37877 {}] procedure2.ProcedureExecutor(1139): Stored pid=60, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=60, table=TestAcidGuarantees 2024-11-16T09:58:59,743 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=037c58a19ddb,37267,1731751077844, seqNum=-1] 2024-11-16T09:58:59,743 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-16T09:58:59,743 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=037c58a19ddb,37267,1731751077844, seqNum=-1] 2024-11-16T09:58:59,743 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:34342, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-16T09:58:59,743 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=037c58a19ddb,37267,1731751077844, seqNum=-1] 2024-11-16T09:58:59,743 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:34368, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-16T09:58:59,743 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-16T09:58:59,743 DEBUG [Thread-1181 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-16T09:58:59,743 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-16T09:58:59,743 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37877 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=60 2024-11-16T09:58:59,744 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:34354, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-16T09:58:59,744 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=037c58a19ddb,37267,1731751077844, seqNum=-1] 2024-11-16T09:58:59,744 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=037c58a19ddb,37267,1731751077844, seqNum=-1] 2024-11-16T09:58:59,744 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:34382, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-16T09:58:59,744 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-16T09:58:59,744 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-16T09:58:59,745 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:34386, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-16T09:58:59,745 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=60, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=60, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-16T09:58:59,745 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:34388, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-16T09:58:59,745 DEBUG [Thread-1185 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-16T09:58:59,746 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:34408, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-16T09:58:59,746 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1731751137553.f3ed23ac403772e0fe4ca9ee892544ec., hostname=037c58a19ddb,37267,1731751077844, seqNum=2] 2024-11-16T09:58:59,746 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1731751137553.f3ed23ac403772e0fe4ca9ee892544ec., hostname=037c58a19ddb,37267,1731751077844, seqNum=2] 2024-11-16T09:58:59,746 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:34404, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-16T09:58:59,746 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1731751137553.f3ed23ac403772e0fe4ca9ee892544ec., hostname=037c58a19ddb,37267,1731751077844, seqNum=2] 2024-11-16T09:58:59,747 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1731751137553.f3ed23ac403772e0fe4ca9ee892544ec., hostname=037c58a19ddb,37267,1731751077844, seqNum=2] 2024-11-16T09:58:59,747 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1731751137553.f3ed23ac403772e0fe4ca9ee892544ec., hostname=037c58a19ddb,37267,1731751077844, seqNum=2] 2024-11-16T09:58:59,747 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=60, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=60, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-16T09:58:59,747 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=61, ppid=60, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-16T09:58:59,747 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1731751137553.f3ed23ac403772e0fe4ca9ee892544ec., hostname=037c58a19ddb,37267,1731751077844, seqNum=2] 2024-11-16T09:58:59,748 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=037c58a19ddb,37267,1731751077844, seqNum=-1] 2024-11-16T09:58:59,748 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1731751137553.f3ed23ac403772e0fe4ca9ee892544ec., hostname=037c58a19ddb,37267,1731751077844, seqNum=2] 2024-11-16T09:58:59,748 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-16T09:58:59,748 DEBUG [Thread-1190 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-16T09:58:59,749 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1731751137553.f3ed23ac403772e0fe4ca9ee892544ec., hostname=037c58a19ddb,37267,1731751077844, seqNum=2] 2024-11-16T09:58:59,749 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:34414, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-16T09:58:59,749 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=037c58a19ddb,37267,1731751077844, seqNum=-1] 2024-11-16T09:58:59,749 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-16T09:58:59,750 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1731751137553.f3ed23ac403772e0fe4ca9ee892544ec., hostname=037c58a19ddb,37267,1731751077844, seqNum=2] 2024-11-16T09:58:59,750 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:34422, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-16T09:58:59,752 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1731751137553.f3ed23ac403772e0fe4ca9ee892544ec., hostname=037c58a19ddb,37267,1731751077844, seqNum=2] 2024-11-16T09:58:59,755 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] regionserver.HRegion(8855): Flush requested on f3ed23ac403772e0fe4ca9ee892544ec 2024-11-16T09:58:59,755 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-16T09:58:59,755 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing f3ed23ac403772e0fe4ca9ee892544ec 3/3 column families, dataSize=53.67 KB heapSize=141.38 KB 2024-11-16T09:58:59,782 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/f3ed23ac403772e0fe4ca9ee892544ec/.tmp/A/c97cbf9e65af4cc4afa063288dd53d11 is 50, key is test_row_0/A:col10/1731751139749/Put/seqid=0 2024-11-16T09:58:59,785 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3ed23ac403772e0fe4ca9ee892544ec, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T09:58:59,785 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34408 deadline: 1731751199779, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3ed23ac403772e0fe4ca9ee892544ec, server=037c58a19ddb,37267,1731751077844 2024-11-16T09:58:59,785 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731751137553.f3ed23ac403772e0fe4ca9ee892544ec., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 , the old value is region=TestAcidGuarantees,,1731751137553.f3ed23ac403772e0fe4ca9ee892544ec., hostname=037c58a19ddb,37267,1731751077844, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3ed23ac403772e0fe4ca9ee892544ec, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:58:59,785 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731751137553.f3ed23ac403772e0fe4ca9ee892544ec., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3ed23ac403772e0fe4ca9ee892544ec, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:58:59,785 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731751137553.f3ed23ac403772e0fe4ca9ee892544ec., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 because the exception is null or not the one we care about 2024-11-16T09:58:59,785 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3ed23ac403772e0fe4ca9ee892544ec, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T09:58:59,785 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] ipc.CallRunner(138): callId: 5 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34414 deadline: 1731751199780, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3ed23ac403772e0fe4ca9ee892544ec, server=037c58a19ddb,37267,1731751077844 2024-11-16T09:58:59,786 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731751137553.f3ed23ac403772e0fe4ca9ee892544ec., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 , the old value is region=TestAcidGuarantees,,1731751137553.f3ed23ac403772e0fe4ca9ee892544ec., hostname=037c58a19ddb,37267,1731751077844, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3ed23ac403772e0fe4ca9ee892544ec, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:58:59,786 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3ed23ac403772e0fe4ca9ee892544ec, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T09:58:59,786 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731751137553.f3ed23ac403772e0fe4ca9ee892544ec., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3ed23ac403772e0fe4ca9ee892544ec, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:58:59,786 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731751137553.f3ed23ac403772e0fe4ca9ee892544ec., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 because the exception is null or not the one we care about 2024-11-16T09:58:59,786 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34382 deadline: 1731751199780, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3ed23ac403772e0fe4ca9ee892544ec, server=037c58a19ddb,37267,1731751077844 2024-11-16T09:58:59,787 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3ed23ac403772e0fe4ca9ee892544ec, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T09:58:59,787 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731751137553.f3ed23ac403772e0fe4ca9ee892544ec., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 , the old value is region=TestAcidGuarantees,,1731751137553.f3ed23ac403772e0fe4ca9ee892544ec., hostname=037c58a19ddb,37267,1731751077844, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3ed23ac403772e0fe4ca9ee892544ec, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:58:59,787 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34386 deadline: 1731751199783, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3ed23ac403772e0fe4ca9ee892544ec, server=037c58a19ddb,37267,1731751077844 2024-11-16T09:58:59,787 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731751137553.f3ed23ac403772e0fe4ca9ee892544ec., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3ed23ac403772e0fe4ca9ee892544ec, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:58:59,787 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731751137553.f3ed23ac403772e0fe4ca9ee892544ec., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 because the exception is null or not the one we care about 2024-11-16T09:58:59,787 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731751137553.f3ed23ac403772e0fe4ca9ee892544ec., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 , the old value is region=TestAcidGuarantees,,1731751137553.f3ed23ac403772e0fe4ca9ee892544ec., hostname=037c58a19ddb,37267,1731751077844, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3ed23ac403772e0fe4ca9ee892544ec, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:58:59,787 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731751137553.f3ed23ac403772e0fe4ca9ee892544ec., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3ed23ac403772e0fe4ca9ee892544ec, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:58:59,787 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731751137553.f3ed23ac403772e0fe4ca9ee892544ec., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 because the exception is null or not the one we care about 2024-11-16T09:58:59,790 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3ed23ac403772e0fe4ca9ee892544ec, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T09:58:59,790 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34368 deadline: 1731751199785, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3ed23ac403772e0fe4ca9ee892544ec, server=037c58a19ddb,37267,1731751077844 2024-11-16T09:58:59,790 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731751137553.f3ed23ac403772e0fe4ca9ee892544ec., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 , the old value is region=TestAcidGuarantees,,1731751137553.f3ed23ac403772e0fe4ca9ee892544ec., hostname=037c58a19ddb,37267,1731751077844, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3ed23ac403772e0fe4ca9ee892544ec, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:58:59,790 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731751137553.f3ed23ac403772e0fe4ca9ee892544ec., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3ed23ac403772e0fe4ca9ee892544ec, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:58:59,790 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731751137553.f3ed23ac403772e0fe4ca9ee892544ec., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 because the exception is null or not the one we care about 2024-11-16T09:58:59,805 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37591 is added to blk_1073742079_1255 (size=12001) 2024-11-16T09:58:59,807 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=13 (bloomFilter=true), to=hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/f3ed23ac403772e0fe4ca9ee892544ec/.tmp/A/c97cbf9e65af4cc4afa063288dd53d11 2024-11-16T09:58:59,836 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/f3ed23ac403772e0fe4ca9ee892544ec/.tmp/B/121e6e78f34e432d92caf3c87f619196 is 50, key is test_row_0/B:col10/1731751139749/Put/seqid=0 2024-11-16T09:58:59,845 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37591 is added to blk_1073742080_1256 (size=12001) 2024-11-16T09:58:59,853 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37877 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=60 2024-11-16T09:58:59,896 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3ed23ac403772e0fe4ca9ee892544ec, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T09:58:59,896 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] ipc.CallRunner(138): callId: 6 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34414 deadline: 1731751199894, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3ed23ac403772e0fe4ca9ee892544ec, server=037c58a19ddb,37267,1731751077844 2024-11-16T09:58:59,896 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3ed23ac403772e0fe4ca9ee892544ec, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T09:58:59,897 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731751137553.f3ed23ac403772e0fe4ca9ee892544ec., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 , the old value is region=TestAcidGuarantees,,1731751137553.f3ed23ac403772e0fe4ca9ee892544ec., hostname=037c58a19ddb,37267,1731751077844, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3ed23ac403772e0fe4ca9ee892544ec, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:58:59,897 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34386 deadline: 1731751199894, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3ed23ac403772e0fe4ca9ee892544ec, server=037c58a19ddb,37267,1731751077844 2024-11-16T09:58:59,897 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731751137553.f3ed23ac403772e0fe4ca9ee892544ec., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3ed23ac403772e0fe4ca9ee892544ec, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:58:59,897 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731751137553.f3ed23ac403772e0fe4ca9ee892544ec., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 because the exception is null or not the one we care about 2024-11-16T09:58:59,897 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731751137553.f3ed23ac403772e0fe4ca9ee892544ec., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 , the old value is region=TestAcidGuarantees,,1731751137553.f3ed23ac403772e0fe4ca9ee892544ec., hostname=037c58a19ddb,37267,1731751077844, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3ed23ac403772e0fe4ca9ee892544ec, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:58:59,897 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731751137553.f3ed23ac403772e0fe4ca9ee892544ec., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3ed23ac403772e0fe4ca9ee892544ec, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:58:59,897 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3ed23ac403772e0fe4ca9ee892544ec, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T09:58:59,897 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731751137553.f3ed23ac403772e0fe4ca9ee892544ec., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 because the exception is null or not the one we care about 2024-11-16T09:58:59,897 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34408 deadline: 1731751199894, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3ed23ac403772e0fe4ca9ee892544ec, server=037c58a19ddb,37267,1731751077844 2024-11-16T09:58:59,897 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731751137553.f3ed23ac403772e0fe4ca9ee892544ec., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 , the old value is region=TestAcidGuarantees,,1731751137553.f3ed23ac403772e0fe4ca9ee892544ec., hostname=037c58a19ddb,37267,1731751077844, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3ed23ac403772e0fe4ca9ee892544ec, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:58:59,897 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3ed23ac403772e0fe4ca9ee892544ec, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T09:58:59,897 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3ed23ac403772e0fe4ca9ee892544ec, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T09:58:59,897 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731751137553.f3ed23ac403772e0fe4ca9ee892544ec., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3ed23ac403772e0fe4ca9ee892544ec, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:58:59,897 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731751137553.f3ed23ac403772e0fe4ca9ee892544ec., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 because the exception is null or not the one we care about 2024-11-16T09:58:59,897 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34368 deadline: 1731751199894, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3ed23ac403772e0fe4ca9ee892544ec, server=037c58a19ddb,37267,1731751077844 2024-11-16T09:58:59,897 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34382 deadline: 1731751199894, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3ed23ac403772e0fe4ca9ee892544ec, server=037c58a19ddb,37267,1731751077844 2024-11-16T09:58:59,898 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731751137553.f3ed23ac403772e0fe4ca9ee892544ec., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 , the old value is region=TestAcidGuarantees,,1731751137553.f3ed23ac403772e0fe4ca9ee892544ec., hostname=037c58a19ddb,37267,1731751077844, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3ed23ac403772e0fe4ca9ee892544ec, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:58:59,898 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731751137553.f3ed23ac403772e0fe4ca9ee892544ec., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 , the old value is region=TestAcidGuarantees,,1731751137553.f3ed23ac403772e0fe4ca9ee892544ec., hostname=037c58a19ddb,37267,1731751077844, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3ed23ac403772e0fe4ca9ee892544ec, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:58:59,898 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731751137553.f3ed23ac403772e0fe4ca9ee892544ec., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3ed23ac403772e0fe4ca9ee892544ec, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:58:59,898 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731751137553.f3ed23ac403772e0fe4ca9ee892544ec., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3ed23ac403772e0fe4ca9ee892544ec, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:58:59,898 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731751137553.f3ed23ac403772e0fe4ca9ee892544ec., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 because the exception is null or not the one we care about 2024-11-16T09:58:59,898 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731751137553.f3ed23ac403772e0fe4ca9ee892544ec., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 because the exception is null or not the one we care about 2024-11-16T09:58:59,902 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=37267 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=61 2024-11-16T09:58:59,902 DEBUG [RS_FLUSH_OPERATIONS-regionserver/037c58a19ddb:0-2 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731751137553.f3ed23ac403772e0fe4ca9ee892544ec. 2024-11-16T09:58:59,902 DEBUG [RS_FLUSH_OPERATIONS-regionserver/037c58a19ddb:0-2 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731751137553.f3ed23ac403772e0fe4ca9ee892544ec. as already flushing 2024-11-16T09:58:59,902 DEBUG [RS_FLUSH_OPERATIONS-regionserver/037c58a19ddb:0-2 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731751137553.f3ed23ac403772e0fe4ca9ee892544ec. 2024-11-16T09:58:59,902 ERROR [RS_FLUSH_OPERATIONS-regionserver/037c58a19ddb:0-2 {event_type=RS_FLUSH_REGIONS, pid=61}] handler.RSProcedureHandler(58): pid=61 java.io.IOException: Unable to complete flush {ENCODED => f3ed23ac403772e0fe4ca9ee892544ec, NAME => 'TestAcidGuarantees,,1731751137553.f3ed23ac403772e0fe4ca9ee892544ec.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T09:58:59,902 DEBUG [RS_FLUSH_OPERATIONS-regionserver/037c58a19ddb:0-2 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=61 java.io.IOException: Unable to complete flush {ENCODED => f3ed23ac403772e0fe4ca9ee892544ec, NAME => 'TestAcidGuarantees,,1731751137553.f3ed23ac403772e0fe4ca9ee892544ec.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T09:58:59,903 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37877 {}] master.HMaster(4177): Remote procedure failed, pid=61 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => f3ed23ac403772e0fe4ca9ee892544ec, NAME => 'TestAcidGuarantees,,1731751137553.f3ed23ac403772e0fe4ca9ee892544ec.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => f3ed23ac403772e0fe4ca9ee892544ec, NAME => 'TestAcidGuarantees,,1731751137553.f3ed23ac403772e0fe4ca9ee892544ec.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T09:59:00,056 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=37267 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=61 2024-11-16T09:59:00,056 DEBUG [RS_FLUSH_OPERATIONS-regionserver/037c58a19ddb:0-0 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731751137553.f3ed23ac403772e0fe4ca9ee892544ec. 2024-11-16T09:59:00,056 DEBUG [RS_FLUSH_OPERATIONS-regionserver/037c58a19ddb:0-0 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731751137553.f3ed23ac403772e0fe4ca9ee892544ec. as already flushing 2024-11-16T09:59:00,056 DEBUG [RS_FLUSH_OPERATIONS-regionserver/037c58a19ddb:0-0 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731751137553.f3ed23ac403772e0fe4ca9ee892544ec. 2024-11-16T09:59:00,056 ERROR [RS_FLUSH_OPERATIONS-regionserver/037c58a19ddb:0-0 {event_type=RS_FLUSH_REGIONS, pid=61}] handler.RSProcedureHandler(58): pid=61 java.io.IOException: Unable to complete flush {ENCODED => f3ed23ac403772e0fe4ca9ee892544ec, NAME => 'TestAcidGuarantees,,1731751137553.f3ed23ac403772e0fe4ca9ee892544ec.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T09:59:00,056 DEBUG [RS_FLUSH_OPERATIONS-regionserver/037c58a19ddb:0-0 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=61 java.io.IOException: Unable to complete flush {ENCODED => f3ed23ac403772e0fe4ca9ee892544ec, NAME => 'TestAcidGuarantees,,1731751137553.f3ed23ac403772e0fe4ca9ee892544ec.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T09:59:00,057 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37877 {}] master.HMaster(4177): Remote procedure failed, pid=61 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => f3ed23ac403772e0fe4ca9ee892544ec, NAME => 'TestAcidGuarantees,,1731751137553.f3ed23ac403772e0fe4ca9ee892544ec.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => f3ed23ac403772e0fe4ca9ee892544ec, NAME => 'TestAcidGuarantees,,1731751137553.f3ed23ac403772e0fe4ca9ee892544ec.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T09:59:00,063 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37877 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=60 2024-11-16T09:59:00,104 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3ed23ac403772e0fe4ca9ee892544ec, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T09:59:00,105 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34414 deadline: 1731751200103, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3ed23ac403772e0fe4ca9ee892544ec, server=037c58a19ddb,37267,1731751077844 2024-11-16T09:59:00,105 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731751137553.f3ed23ac403772e0fe4ca9ee892544ec., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 , the old value is region=TestAcidGuarantees,,1731751137553.f3ed23ac403772e0fe4ca9ee892544ec., hostname=037c58a19ddb,37267,1731751077844, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3ed23ac403772e0fe4ca9ee892544ec, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:59:00,105 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731751137553.f3ed23ac403772e0fe4ca9ee892544ec., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3ed23ac403772e0fe4ca9ee892544ec, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:59:00,105 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731751137553.f3ed23ac403772e0fe4ca9ee892544ec., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 because the exception is null or not the one we care about 2024-11-16T09:59:00,105 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3ed23ac403772e0fe4ca9ee892544ec, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T09:59:00,105 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34386 deadline: 1731751200103, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3ed23ac403772e0fe4ca9ee892544ec, server=037c58a19ddb,37267,1731751077844 2024-11-16T09:59:00,105 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3ed23ac403772e0fe4ca9ee892544ec, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T09:59:00,105 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731751137553.f3ed23ac403772e0fe4ca9ee892544ec., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 , the old value is region=TestAcidGuarantees,,1731751137553.f3ed23ac403772e0fe4ca9ee892544ec., hostname=037c58a19ddb,37267,1731751077844, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3ed23ac403772e0fe4ca9ee892544ec, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:59:00,105 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34368 deadline: 1731751200103, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3ed23ac403772e0fe4ca9ee892544ec, server=037c58a19ddb,37267,1731751077844 2024-11-16T09:59:00,105 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731751137553.f3ed23ac403772e0fe4ca9ee892544ec., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3ed23ac403772e0fe4ca9ee892544ec, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:59:00,106 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731751137553.f3ed23ac403772e0fe4ca9ee892544ec., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 because the exception is null or not the one we care about 2024-11-16T09:59:00,106 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731751137553.f3ed23ac403772e0fe4ca9ee892544ec., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 , the old value is region=TestAcidGuarantees,,1731751137553.f3ed23ac403772e0fe4ca9ee892544ec., hostname=037c58a19ddb,37267,1731751077844, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3ed23ac403772e0fe4ca9ee892544ec, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:59:00,106 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3ed23ac403772e0fe4ca9ee892544ec, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T09:59:00,106 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731751137553.f3ed23ac403772e0fe4ca9ee892544ec., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3ed23ac403772e0fe4ca9ee892544ec, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:59:00,106 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731751137553.f3ed23ac403772e0fe4ca9ee892544ec., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 because the exception is null or not the one we care about 2024-11-16T09:59:00,106 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34408 deadline: 1731751200103, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3ed23ac403772e0fe4ca9ee892544ec, server=037c58a19ddb,37267,1731751077844 2024-11-16T09:59:00,106 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3ed23ac403772e0fe4ca9ee892544ec, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T09:59:00,106 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34382 deadline: 1731751200103, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3ed23ac403772e0fe4ca9ee892544ec, server=037c58a19ddb,37267,1731751077844 2024-11-16T09:59:00,106 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731751137553.f3ed23ac403772e0fe4ca9ee892544ec., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 , the old value is region=TestAcidGuarantees,,1731751137553.f3ed23ac403772e0fe4ca9ee892544ec., hostname=037c58a19ddb,37267,1731751077844, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3ed23ac403772e0fe4ca9ee892544ec, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:59:00,106 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731751137553.f3ed23ac403772e0fe4ca9ee892544ec., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3ed23ac403772e0fe4ca9ee892544ec, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:59:00,106 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731751137553.f3ed23ac403772e0fe4ca9ee892544ec., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 because the exception is null or not the one we care about 2024-11-16T09:59:00,106 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731751137553.f3ed23ac403772e0fe4ca9ee892544ec., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 , the old value is region=TestAcidGuarantees,,1731751137553.f3ed23ac403772e0fe4ca9ee892544ec., hostname=037c58a19ddb,37267,1731751077844, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3ed23ac403772e0fe4ca9ee892544ec, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:59:00,106 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731751137553.f3ed23ac403772e0fe4ca9ee892544ec., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3ed23ac403772e0fe4ca9ee892544ec, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:59:00,106 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731751137553.f3ed23ac403772e0fe4ca9ee892544ec., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 because the exception is null or not the one we care about 2024-11-16T09:59:00,209 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=37267 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=61 2024-11-16T09:59:00,209 DEBUG [RS_FLUSH_OPERATIONS-regionserver/037c58a19ddb:0-1 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731751137553.f3ed23ac403772e0fe4ca9ee892544ec. 2024-11-16T09:59:00,209 DEBUG [RS_FLUSH_OPERATIONS-regionserver/037c58a19ddb:0-1 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731751137553.f3ed23ac403772e0fe4ca9ee892544ec. as already flushing 2024-11-16T09:59:00,209 DEBUG [RS_FLUSH_OPERATIONS-regionserver/037c58a19ddb:0-1 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731751137553.f3ed23ac403772e0fe4ca9ee892544ec. 2024-11-16T09:59:00,209 ERROR [RS_FLUSH_OPERATIONS-regionserver/037c58a19ddb:0-1 {event_type=RS_FLUSH_REGIONS, pid=61}] handler.RSProcedureHandler(58): pid=61 java.io.IOException: Unable to complete flush {ENCODED => f3ed23ac403772e0fe4ca9ee892544ec, NAME => 'TestAcidGuarantees,,1731751137553.f3ed23ac403772e0fe4ca9ee892544ec.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T09:59:00,210 DEBUG [RS_FLUSH_OPERATIONS-regionserver/037c58a19ddb:0-1 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=61 java.io.IOException: Unable to complete flush {ENCODED => f3ed23ac403772e0fe4ca9ee892544ec, NAME => 'TestAcidGuarantees,,1731751137553.f3ed23ac403772e0fe4ca9ee892544ec.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T09:59:00,210 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37877 {}] master.HMaster(4177): Remote procedure failed, pid=61 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => f3ed23ac403772e0fe4ca9ee892544ec, NAME => 'TestAcidGuarantees,,1731751137553.f3ed23ac403772e0fe4ca9ee892544ec.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => f3ed23ac403772e0fe4ca9ee892544ec, NAME => 'TestAcidGuarantees,,1731751137553.f3ed23ac403772e0fe4ca9ee892544ec.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T09:59:00,246 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=13 (bloomFilter=true), to=hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/f3ed23ac403772e0fe4ca9ee892544ec/.tmp/B/121e6e78f34e432d92caf3c87f619196 2024-11-16T09:59:00,270 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/f3ed23ac403772e0fe4ca9ee892544ec/.tmp/C/f5e10ca7ce7f4643ac9f2b880087ce59 is 50, key is test_row_0/C:col10/1731751139749/Put/seqid=0 2024-11-16T09:59:00,274 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37591 is added to blk_1073742081_1257 (size=12001) 2024-11-16T09:59:00,275 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=13 (bloomFilter=true), to=hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/f3ed23ac403772e0fe4ca9ee892544ec/.tmp/C/f5e10ca7ce7f4643ac9f2b880087ce59 2024-11-16T09:59:00,281 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/f3ed23ac403772e0fe4ca9ee892544ec/.tmp/A/c97cbf9e65af4cc4afa063288dd53d11 as hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/f3ed23ac403772e0fe4ca9ee892544ec/A/c97cbf9e65af4cc4afa063288dd53d11 2024-11-16T09:59:00,287 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/f3ed23ac403772e0fe4ca9ee892544ec/A/c97cbf9e65af4cc4afa063288dd53d11, entries=150, sequenceid=13, filesize=11.7 K 2024-11-16T09:59:00,288 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/f3ed23ac403772e0fe4ca9ee892544ec/.tmp/B/121e6e78f34e432d92caf3c87f619196 as hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/f3ed23ac403772e0fe4ca9ee892544ec/B/121e6e78f34e432d92caf3c87f619196 2024-11-16T09:59:00,293 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/f3ed23ac403772e0fe4ca9ee892544ec/B/121e6e78f34e432d92caf3c87f619196, entries=150, sequenceid=13, filesize=11.7 K 2024-11-16T09:59:00,294 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/f3ed23ac403772e0fe4ca9ee892544ec/.tmp/C/f5e10ca7ce7f4643ac9f2b880087ce59 as hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/f3ed23ac403772e0fe4ca9ee892544ec/C/f5e10ca7ce7f4643ac9f2b880087ce59 2024-11-16T09:59:00,299 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/f3ed23ac403772e0fe4ca9ee892544ec/C/f5e10ca7ce7f4643ac9f2b880087ce59, entries=150, sequenceid=13, filesize=11.7 K 2024-11-16T09:59:00,299 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~158.91 KB/162720, currentSize=140.89 KB/144270 for f3ed23ac403772e0fe4ca9ee892544ec in 544ms, sequenceid=13, compaction requested=false 2024-11-16T09:59:00,300 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for f3ed23ac403772e0fe4ca9ee892544ec: 2024-11-16T09:59:00,362 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=37267 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=61 2024-11-16T09:59:00,362 DEBUG [RS_FLUSH_OPERATIONS-regionserver/037c58a19ddb:0-2 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731751137553.f3ed23ac403772e0fe4ca9ee892544ec. 2024-11-16T09:59:00,363 INFO [RS_FLUSH_OPERATIONS-regionserver/037c58a19ddb:0-2 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.HRegion(2902): Flushing f3ed23ac403772e0fe4ca9ee892544ec 3/3 column families, dataSize=140.89 KB heapSize=369.89 KB 2024-11-16T09:59:00,368 DEBUG [RS_FLUSH_OPERATIONS-regionserver/037c58a19ddb:0-2 {event_type=RS_FLUSH_REGIONS, pid=61}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/f3ed23ac403772e0fe4ca9ee892544ec/.tmp/A/c3e74913af05458d9452010d8e8044c0 is 50, key is test_row_0/A:col10/1731751139767/Put/seqid=0 2024-11-16T09:59:00,372 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37591 is added to blk_1073742082_1258 (size=12001) 2024-11-16T09:59:00,373 INFO [RS_FLUSH_OPERATIONS-regionserver/037c58a19ddb:0-2 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=37 (bloomFilter=true), to=hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/f3ed23ac403772e0fe4ca9ee892544ec/.tmp/A/c3e74913af05458d9452010d8e8044c0 2024-11-16T09:59:00,373 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37877 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=60 2024-11-16T09:59:00,382 DEBUG [RS_FLUSH_OPERATIONS-regionserver/037c58a19ddb:0-2 {event_type=RS_FLUSH_REGIONS, pid=61}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/f3ed23ac403772e0fe4ca9ee892544ec/.tmp/B/6b4c1d88775e40b89a0762d4905b7d9f is 50, key is test_row_0/B:col10/1731751139767/Put/seqid=0 2024-11-16T09:59:00,416 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] regionserver.HRegion(8855): Flush requested on f3ed23ac403772e0fe4ca9ee892544ec 2024-11-16T09:59:00,416 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731751137553.f3ed23ac403772e0fe4ca9ee892544ec. as already flushing 2024-11-16T09:59:00,425 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3ed23ac403772e0fe4ca9ee892544ec, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T09:59:00,425 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34382 deadline: 1731751200419, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3ed23ac403772e0fe4ca9ee892544ec, server=037c58a19ddb,37267,1731751077844 2024-11-16T09:59:00,425 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731751137553.f3ed23ac403772e0fe4ca9ee892544ec., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 , the old value is region=TestAcidGuarantees,,1731751137553.f3ed23ac403772e0fe4ca9ee892544ec., hostname=037c58a19ddb,37267,1731751077844, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3ed23ac403772e0fe4ca9ee892544ec, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:59:00,425 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3ed23ac403772e0fe4ca9ee892544ec, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T09:59:00,426 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731751137553.f3ed23ac403772e0fe4ca9ee892544ec., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3ed23ac403772e0fe4ca9ee892544ec, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:59:00,426 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731751137553.f3ed23ac403772e0fe4ca9ee892544ec., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 because the exception is null or not the one we care about 2024-11-16T09:59:00,426 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34386 deadline: 1731751200422, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3ed23ac403772e0fe4ca9ee892544ec, server=037c58a19ddb,37267,1731751077844 2024-11-16T09:59:00,426 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731751137553.f3ed23ac403772e0fe4ca9ee892544ec., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 , the old value is region=TestAcidGuarantees,,1731751137553.f3ed23ac403772e0fe4ca9ee892544ec., hostname=037c58a19ddb,37267,1731751077844, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3ed23ac403772e0fe4ca9ee892544ec, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:59:00,426 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731751137553.f3ed23ac403772e0fe4ca9ee892544ec., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3ed23ac403772e0fe4ca9ee892544ec, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:59:00,426 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731751137553.f3ed23ac403772e0fe4ca9ee892544ec., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 because the exception is null or not the one we care about 2024-11-16T09:59:00,427 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3ed23ac403772e0fe4ca9ee892544ec, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T09:59:00,427 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34408 deadline: 1731751200425, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3ed23ac403772e0fe4ca9ee892544ec, server=037c58a19ddb,37267,1731751077844 2024-11-16T09:59:00,428 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731751137553.f3ed23ac403772e0fe4ca9ee892544ec., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 , the old value is region=TestAcidGuarantees,,1731751137553.f3ed23ac403772e0fe4ca9ee892544ec., hostname=037c58a19ddb,37267,1731751077844, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3ed23ac403772e0fe4ca9ee892544ec, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:59:00,428 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731751137553.f3ed23ac403772e0fe4ca9ee892544ec., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3ed23ac403772e0fe4ca9ee892544ec, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:59:00,428 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731751137553.f3ed23ac403772e0fe4ca9ee892544ec., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 because the exception is null or not the one we care about 2024-11-16T09:59:00,433 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3ed23ac403772e0fe4ca9ee892544ec, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T09:59:00,433 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37591 is added to blk_1073742083_1259 (size=12001) 2024-11-16T09:59:00,434 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34414 deadline: 1731751200432, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3ed23ac403772e0fe4ca9ee892544ec, server=037c58a19ddb,37267,1731751077844 2024-11-16T09:59:00,434 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731751137553.f3ed23ac403772e0fe4ca9ee892544ec., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 , the old value is region=TestAcidGuarantees,,1731751137553.f3ed23ac403772e0fe4ca9ee892544ec., hostname=037c58a19ddb,37267,1731751077844, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3ed23ac403772e0fe4ca9ee892544ec, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:59:00,434 INFO [RS_FLUSH_OPERATIONS-regionserver/037c58a19ddb:0-2 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=37 (bloomFilter=true), to=hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/f3ed23ac403772e0fe4ca9ee892544ec/.tmp/B/6b4c1d88775e40b89a0762d4905b7d9f 2024-11-16T09:59:00,434 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3ed23ac403772e0fe4ca9ee892544ec, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T09:59:00,434 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731751137553.f3ed23ac403772e0fe4ca9ee892544ec., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3ed23ac403772e0fe4ca9ee892544ec, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:59:00,434 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731751137553.f3ed23ac403772e0fe4ca9ee892544ec., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 because the exception is null or not the one we care about 2024-11-16T09:59:00,434 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34368 deadline: 1731751200432, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3ed23ac403772e0fe4ca9ee892544ec, server=037c58a19ddb,37267,1731751077844 2024-11-16T09:59:00,435 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731751137553.f3ed23ac403772e0fe4ca9ee892544ec., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 , the old value is region=TestAcidGuarantees,,1731751137553.f3ed23ac403772e0fe4ca9ee892544ec., hostname=037c58a19ddb,37267,1731751077844, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3ed23ac403772e0fe4ca9ee892544ec, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:59:00,435 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731751137553.f3ed23ac403772e0fe4ca9ee892544ec., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3ed23ac403772e0fe4ca9ee892544ec, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:59:00,435 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731751137553.f3ed23ac403772e0fe4ca9ee892544ec., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 because the exception is null or not the one we care about 2024-11-16T09:59:00,442 DEBUG [RS_FLUSH_OPERATIONS-regionserver/037c58a19ddb:0-2 {event_type=RS_FLUSH_REGIONS, pid=61}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/f3ed23ac403772e0fe4ca9ee892544ec/.tmp/C/03ddda3dad844fc09f3e33be2515e85e is 50, key is test_row_0/C:col10/1731751139767/Put/seqid=0 2024-11-16T09:59:00,447 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37591 is added to blk_1073742084_1260 (size=12001) 2024-11-16T09:59:00,534 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3ed23ac403772e0fe4ca9ee892544ec, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T09:59:00,534 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34382 deadline: 1731751200532, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3ed23ac403772e0fe4ca9ee892544ec, server=037c58a19ddb,37267,1731751077844 2024-11-16T09:59:00,535 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3ed23ac403772e0fe4ca9ee892544ec, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T09:59:00,535 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34386 deadline: 1731751200532, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3ed23ac403772e0fe4ca9ee892544ec, server=037c58a19ddb,37267,1731751077844 2024-11-16T09:59:00,535 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731751137553.f3ed23ac403772e0fe4ca9ee892544ec., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 , the old value is region=TestAcidGuarantees,,1731751137553.f3ed23ac403772e0fe4ca9ee892544ec., hostname=037c58a19ddb,37267,1731751077844, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3ed23ac403772e0fe4ca9ee892544ec, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:59:00,535 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731751137553.f3ed23ac403772e0fe4ca9ee892544ec., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3ed23ac403772e0fe4ca9ee892544ec, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:59:00,535 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731751137553.f3ed23ac403772e0fe4ca9ee892544ec., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 because the exception is null or not the one we care about 2024-11-16T09:59:00,535 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3ed23ac403772e0fe4ca9ee892544ec, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T09:59:00,535 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731751137553.f3ed23ac403772e0fe4ca9ee892544ec., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 , the old value is region=TestAcidGuarantees,,1731751137553.f3ed23ac403772e0fe4ca9ee892544ec., hostname=037c58a19ddb,37267,1731751077844, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3ed23ac403772e0fe4ca9ee892544ec, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:59:00,535 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34408 deadline: 1731751200532, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3ed23ac403772e0fe4ca9ee892544ec, server=037c58a19ddb,37267,1731751077844 2024-11-16T09:59:00,535 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731751137553.f3ed23ac403772e0fe4ca9ee892544ec., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3ed23ac403772e0fe4ca9ee892544ec, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:59:00,535 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731751137553.f3ed23ac403772e0fe4ca9ee892544ec., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 because the exception is null or not the one we care about 2024-11-16T09:59:00,535 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731751137553.f3ed23ac403772e0fe4ca9ee892544ec., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 , the old value is region=TestAcidGuarantees,,1731751137553.f3ed23ac403772e0fe4ca9ee892544ec., hostname=037c58a19ddb,37267,1731751077844, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3ed23ac403772e0fe4ca9ee892544ec, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:59:00,535 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731751137553.f3ed23ac403772e0fe4ca9ee892544ec., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3ed23ac403772e0fe4ca9ee892544ec, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:59:00,535 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731751137553.f3ed23ac403772e0fe4ca9ee892544ec., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 because the exception is null or not the one we care about 2024-11-16T09:59:00,544 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3ed23ac403772e0fe4ca9ee892544ec, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T09:59:00,544 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34414 deadline: 1731751200543, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3ed23ac403772e0fe4ca9ee892544ec, server=037c58a19ddb,37267,1731751077844 2024-11-16T09:59:00,544 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731751137553.f3ed23ac403772e0fe4ca9ee892544ec., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 , the old value is region=TestAcidGuarantees,,1731751137553.f3ed23ac403772e0fe4ca9ee892544ec., hostname=037c58a19ddb,37267,1731751077844, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3ed23ac403772e0fe4ca9ee892544ec, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:59:00,544 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731751137553.f3ed23ac403772e0fe4ca9ee892544ec., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3ed23ac403772e0fe4ca9ee892544ec, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:59:00,544 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3ed23ac403772e0fe4ca9ee892544ec, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T09:59:00,544 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731751137553.f3ed23ac403772e0fe4ca9ee892544ec., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 because the exception is null or not the one we care about 2024-11-16T09:59:00,545 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34368 deadline: 1731751200543, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3ed23ac403772e0fe4ca9ee892544ec, server=037c58a19ddb,37267,1731751077844 2024-11-16T09:59:00,545 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731751137553.f3ed23ac403772e0fe4ca9ee892544ec., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 , the old value is region=TestAcidGuarantees,,1731751137553.f3ed23ac403772e0fe4ca9ee892544ec., hostname=037c58a19ddb,37267,1731751077844, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3ed23ac403772e0fe4ca9ee892544ec, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:59:00,545 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731751137553.f3ed23ac403772e0fe4ca9ee892544ec., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3ed23ac403772e0fe4ca9ee892544ec, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:59:00,545 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731751137553.f3ed23ac403772e0fe4ca9ee892544ec., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 because the exception is null or not the one we care about 2024-11-16T09:59:00,746 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3ed23ac403772e0fe4ca9ee892544ec, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T09:59:00,746 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34382 deadline: 1731751200743, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3ed23ac403772e0fe4ca9ee892544ec, server=037c58a19ddb,37267,1731751077844 2024-11-16T09:59:00,747 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3ed23ac403772e0fe4ca9ee892544ec, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T09:59:00,747 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3ed23ac403772e0fe4ca9ee892544ec, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T09:59:00,747 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34386 deadline: 1731751200743, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3ed23ac403772e0fe4ca9ee892544ec, server=037c58a19ddb,37267,1731751077844 2024-11-16T09:59:00,747 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34408 deadline: 1731751200743, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3ed23ac403772e0fe4ca9ee892544ec, server=037c58a19ddb,37267,1731751077844 2024-11-16T09:59:00,747 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731751137553.f3ed23ac403772e0fe4ca9ee892544ec., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 , the old value is region=TestAcidGuarantees,,1731751137553.f3ed23ac403772e0fe4ca9ee892544ec., hostname=037c58a19ddb,37267,1731751077844, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3ed23ac403772e0fe4ca9ee892544ec, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:59:00,747 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731751137553.f3ed23ac403772e0fe4ca9ee892544ec., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3ed23ac403772e0fe4ca9ee892544ec, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:59:00,747 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731751137553.f3ed23ac403772e0fe4ca9ee892544ec., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 because the exception is null or not the one we care about 2024-11-16T09:59:00,747 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731751137553.f3ed23ac403772e0fe4ca9ee892544ec., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 , the old value is region=TestAcidGuarantees,,1731751137553.f3ed23ac403772e0fe4ca9ee892544ec., hostname=037c58a19ddb,37267,1731751077844, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3ed23ac403772e0fe4ca9ee892544ec, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:59:00,747 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731751137553.f3ed23ac403772e0fe4ca9ee892544ec., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3ed23ac403772e0fe4ca9ee892544ec, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:59:00,747 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731751137553.f3ed23ac403772e0fe4ca9ee892544ec., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 because the exception is null or not the one we care about 2024-11-16T09:59:00,747 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731751137553.f3ed23ac403772e0fe4ca9ee892544ec., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 , the old value is region=TestAcidGuarantees,,1731751137553.f3ed23ac403772e0fe4ca9ee892544ec., hostname=037c58a19ddb,37267,1731751077844, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3ed23ac403772e0fe4ca9ee892544ec, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:59:00,747 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731751137553.f3ed23ac403772e0fe4ca9ee892544ec., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3ed23ac403772e0fe4ca9ee892544ec, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:59:00,747 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731751137553.f3ed23ac403772e0fe4ca9ee892544ec., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 because the exception is null or not the one we care about 2024-11-16T09:59:00,756 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3ed23ac403772e0fe4ca9ee892544ec, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T09:59:00,757 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34414 deadline: 1731751200753, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3ed23ac403772e0fe4ca9ee892544ec, server=037c58a19ddb,37267,1731751077844 2024-11-16T09:59:00,757 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731751137553.f3ed23ac403772e0fe4ca9ee892544ec., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 , the old value is region=TestAcidGuarantees,,1731751137553.f3ed23ac403772e0fe4ca9ee892544ec., hostname=037c58a19ddb,37267,1731751077844, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3ed23ac403772e0fe4ca9ee892544ec, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:59:00,757 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3ed23ac403772e0fe4ca9ee892544ec, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T09:59:00,757 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731751137553.f3ed23ac403772e0fe4ca9ee892544ec., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3ed23ac403772e0fe4ca9ee892544ec, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:59:00,757 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731751137553.f3ed23ac403772e0fe4ca9ee892544ec., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 because the exception is null or not the one we care about 2024-11-16T09:59:00,757 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34368 deadline: 1731751200753, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3ed23ac403772e0fe4ca9ee892544ec, server=037c58a19ddb,37267,1731751077844 2024-11-16T09:59:00,757 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731751137553.f3ed23ac403772e0fe4ca9ee892544ec., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 , the old value is region=TestAcidGuarantees,,1731751137553.f3ed23ac403772e0fe4ca9ee892544ec., hostname=037c58a19ddb,37267,1731751077844, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3ed23ac403772e0fe4ca9ee892544ec, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:59:00,757 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731751137553.f3ed23ac403772e0fe4ca9ee892544ec., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3ed23ac403772e0fe4ca9ee892544ec, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:59:00,757 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731751137553.f3ed23ac403772e0fe4ca9ee892544ec., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 because the exception is null or not the one we care about 2024-11-16T09:59:00,848 INFO [RS_FLUSH_OPERATIONS-regionserver/037c58a19ddb:0-2 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=37 (bloomFilter=true), to=hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/f3ed23ac403772e0fe4ca9ee892544ec/.tmp/C/03ddda3dad844fc09f3e33be2515e85e 2024-11-16T09:59:00,853 DEBUG [RS_FLUSH_OPERATIONS-regionserver/037c58a19ddb:0-2 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/f3ed23ac403772e0fe4ca9ee892544ec/.tmp/A/c3e74913af05458d9452010d8e8044c0 as hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/f3ed23ac403772e0fe4ca9ee892544ec/A/c3e74913af05458d9452010d8e8044c0 2024-11-16T09:59:00,858 INFO [RS_FLUSH_OPERATIONS-regionserver/037c58a19ddb:0-2 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/f3ed23ac403772e0fe4ca9ee892544ec/A/c3e74913af05458d9452010d8e8044c0, entries=150, sequenceid=37, filesize=11.7 K 2024-11-16T09:59:00,859 DEBUG [RS_FLUSH_OPERATIONS-regionserver/037c58a19ddb:0-2 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/f3ed23ac403772e0fe4ca9ee892544ec/.tmp/B/6b4c1d88775e40b89a0762d4905b7d9f as hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/f3ed23ac403772e0fe4ca9ee892544ec/B/6b4c1d88775e40b89a0762d4905b7d9f 2024-11-16T09:59:00,865 INFO [RS_FLUSH_OPERATIONS-regionserver/037c58a19ddb:0-2 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/f3ed23ac403772e0fe4ca9ee892544ec/B/6b4c1d88775e40b89a0762d4905b7d9f, entries=150, sequenceid=37, filesize=11.7 K 2024-11-16T09:59:00,866 DEBUG [RS_FLUSH_OPERATIONS-regionserver/037c58a19ddb:0-2 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/f3ed23ac403772e0fe4ca9ee892544ec/.tmp/C/03ddda3dad844fc09f3e33be2515e85e as hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/f3ed23ac403772e0fe4ca9ee892544ec/C/03ddda3dad844fc09f3e33be2515e85e 2024-11-16T09:59:00,871 INFO [RS_FLUSH_OPERATIONS-regionserver/037c58a19ddb:0-2 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/f3ed23ac403772e0fe4ca9ee892544ec/C/03ddda3dad844fc09f3e33be2515e85e, entries=150, sequenceid=37, filesize=11.7 K 2024-11-16T09:59:00,872 INFO [RS_FLUSH_OPERATIONS-regionserver/037c58a19ddb:0-2 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.HRegion(3140): Finished flush of dataSize ~140.89 KB/144270, heapSize ~369.84 KB/378720, currentSize=60.38 KB/61830 for f3ed23ac403772e0fe4ca9ee892544ec in 509ms, sequenceid=37, compaction requested=false 2024-11-16T09:59:00,872 DEBUG [RS_FLUSH_OPERATIONS-regionserver/037c58a19ddb:0-2 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.HRegion(2603): Flush status journal for f3ed23ac403772e0fe4ca9ee892544ec: 2024-11-16T09:59:00,872 DEBUG [RS_FLUSH_OPERATIONS-regionserver/037c58a19ddb:0-2 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731751137553.f3ed23ac403772e0fe4ca9ee892544ec. 2024-11-16T09:59:00,872 DEBUG [RS_FLUSH_OPERATIONS-regionserver/037c58a19ddb:0-2 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=61 2024-11-16T09:59:00,873 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37877 {}] master.HMaster(4169): Remote procedure done, pid=61 2024-11-16T09:59:00,876 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=61, resume processing ppid=60 2024-11-16T09:59:00,876 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=61, ppid=60, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.1270 sec 2024-11-16T09:59:00,879 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=60, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=60, table=TestAcidGuarantees in 1.1360 sec 2024-11-16T09:59:00,883 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37877 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=60 2024-11-16T09:59:00,883 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-16T09:59:00,884 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37877 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-16T09:59:00,885 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37877 {}] procedure2.ProcedureExecutor(1139): Stored pid=62, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=62, table=TestAcidGuarantees 2024-11-16T09:59:00,886 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37877 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=62 2024-11-16T09:59:00,886 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=62, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=62, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-16T09:59:00,887 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=62, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=62, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-16T09:59:00,887 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=63, ppid=62, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-16T09:59:00,993 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37877 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=62 2024-11-16T09:59:01,056 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=37267 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=63 2024-11-16T09:59:01,056 DEBUG [RS_FLUSH_OPERATIONS-regionserver/037c58a19ddb:0-0 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731751137553.f3ed23ac403772e0fe4ca9ee892544ec. 2024-11-16T09:59:01,056 INFO [RS_FLUSH_OPERATIONS-regionserver/037c58a19ddb:0-0 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.HRegion(2902): Flushing f3ed23ac403772e0fe4ca9ee892544ec 3/3 column families, dataSize=60.38 KB heapSize=158.95 KB 2024-11-16T09:59:01,057 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] regionserver.HRegion(8855): Flush requested on f3ed23ac403772e0fe4ca9ee892544ec 2024-11-16T09:59:01,057 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731751137553.f3ed23ac403772e0fe4ca9ee892544ec. as already flushing 2024-11-16T09:59:01,062 DEBUG [RS_FLUSH_OPERATIONS-regionserver/037c58a19ddb:0-0 {event_type=RS_FLUSH_REGIONS, pid=63}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/f3ed23ac403772e0fe4ca9ee892544ec/.tmp/A/3dc937ecac70408cba3ef81b89802dd1 is 50, key is test_row_0/A:col10/1731751140417/Put/seqid=0 2024-11-16T09:59:01,070 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37591 is added to blk_1073742085_1261 (size=12001) 2024-11-16T09:59:01,115 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3ed23ac403772e0fe4ca9ee892544ec, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T09:59:01,115 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34368 deadline: 1731751201109, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3ed23ac403772e0fe4ca9ee892544ec, server=037c58a19ddb,37267,1731751077844 2024-11-16T09:59:01,115 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3ed23ac403772e0fe4ca9ee892544ec, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T09:59:01,115 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34414 deadline: 1731751201110, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3ed23ac403772e0fe4ca9ee892544ec, server=037c58a19ddb,37267,1731751077844 2024-11-16T09:59:01,115 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3ed23ac403772e0fe4ca9ee892544ec, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T09:59:01,115 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34386 deadline: 1731751201111, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3ed23ac403772e0fe4ca9ee892544ec, server=037c58a19ddb,37267,1731751077844 2024-11-16T09:59:01,116 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3ed23ac403772e0fe4ca9ee892544ec, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T09:59:01,116 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731751137553.f3ed23ac403772e0fe4ca9ee892544ec., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 , the old value is region=TestAcidGuarantees,,1731751137553.f3ed23ac403772e0fe4ca9ee892544ec., hostname=037c58a19ddb,37267,1731751077844, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3ed23ac403772e0fe4ca9ee892544ec, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:59:01,116 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731751137553.f3ed23ac403772e0fe4ca9ee892544ec., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 , the old value is region=TestAcidGuarantees,,1731751137553.f3ed23ac403772e0fe4ca9ee892544ec., hostname=037c58a19ddb,37267,1731751077844, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3ed23ac403772e0fe4ca9ee892544ec, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:59:01,116 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731751137553.f3ed23ac403772e0fe4ca9ee892544ec., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 , the old value is region=TestAcidGuarantees,,1731751137553.f3ed23ac403772e0fe4ca9ee892544ec., hostname=037c58a19ddb,37267,1731751077844, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3ed23ac403772e0fe4ca9ee892544ec, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:59:01,116 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731751137553.f3ed23ac403772e0fe4ca9ee892544ec., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3ed23ac403772e0fe4ca9ee892544ec, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:59:01,116 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731751137553.f3ed23ac403772e0fe4ca9ee892544ec., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3ed23ac403772e0fe4ca9ee892544ec, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:59:01,116 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731751137553.f3ed23ac403772e0fe4ca9ee892544ec., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3ed23ac403772e0fe4ca9ee892544ec, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:59:01,116 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34382 deadline: 1731751201111, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3ed23ac403772e0fe4ca9ee892544ec, server=037c58a19ddb,37267,1731751077844 2024-11-16T09:59:01,116 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731751137553.f3ed23ac403772e0fe4ca9ee892544ec., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 because the exception is null or not the one we care about 2024-11-16T09:59:01,116 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731751137553.f3ed23ac403772e0fe4ca9ee892544ec., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 because the exception is null or not the one we care about 2024-11-16T09:59:01,116 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731751137553.f3ed23ac403772e0fe4ca9ee892544ec., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 because the exception is null or not the one we care about 2024-11-16T09:59:01,116 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731751137553.f3ed23ac403772e0fe4ca9ee892544ec., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 , the old value is region=TestAcidGuarantees,,1731751137553.f3ed23ac403772e0fe4ca9ee892544ec., hostname=037c58a19ddb,37267,1731751077844, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3ed23ac403772e0fe4ca9ee892544ec, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:59:01,116 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731751137553.f3ed23ac403772e0fe4ca9ee892544ec., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3ed23ac403772e0fe4ca9ee892544ec, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:59:01,116 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3ed23ac403772e0fe4ca9ee892544ec, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T09:59:01,116 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731751137553.f3ed23ac403772e0fe4ca9ee892544ec., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 because the exception is null or not the one we care about 2024-11-16T09:59:01,116 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34408 deadline: 1731751201111, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3ed23ac403772e0fe4ca9ee892544ec, server=037c58a19ddb,37267,1731751077844 2024-11-16T09:59:01,117 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731751137553.f3ed23ac403772e0fe4ca9ee892544ec., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 , the old value is region=TestAcidGuarantees,,1731751137553.f3ed23ac403772e0fe4ca9ee892544ec., hostname=037c58a19ddb,37267,1731751077844, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3ed23ac403772e0fe4ca9ee892544ec, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:59:01,117 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731751137553.f3ed23ac403772e0fe4ca9ee892544ec., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3ed23ac403772e0fe4ca9ee892544ec, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:59:01,117 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731751137553.f3ed23ac403772e0fe4ca9ee892544ec., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 because the exception is null or not the one we care about 2024-11-16T09:59:01,203 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37877 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=62 2024-11-16T09:59:01,225 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3ed23ac403772e0fe4ca9ee892544ec, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T09:59:01,225 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34368 deadline: 1731751201223, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3ed23ac403772e0fe4ca9ee892544ec, server=037c58a19ddb,37267,1731751077844 2024-11-16T09:59:01,225 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3ed23ac403772e0fe4ca9ee892544ec, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T09:59:01,226 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731751137553.f3ed23ac403772e0fe4ca9ee892544ec., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 , the old value is region=TestAcidGuarantees,,1731751137553.f3ed23ac403772e0fe4ca9ee892544ec., hostname=037c58a19ddb,37267,1731751077844, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3ed23ac403772e0fe4ca9ee892544ec, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:59:01,226 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731751137553.f3ed23ac403772e0fe4ca9ee892544ec., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3ed23ac403772e0fe4ca9ee892544ec, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:59:01,226 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34414 deadline: 1731751201223, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3ed23ac403772e0fe4ca9ee892544ec, server=037c58a19ddb,37267,1731751077844 2024-11-16T09:59:01,226 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731751137553.f3ed23ac403772e0fe4ca9ee892544ec., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 because the exception is null or not the one we care about 2024-11-16T09:59:01,226 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731751137553.f3ed23ac403772e0fe4ca9ee892544ec., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 , the old value is region=TestAcidGuarantees,,1731751137553.f3ed23ac403772e0fe4ca9ee892544ec., hostname=037c58a19ddb,37267,1731751077844, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3ed23ac403772e0fe4ca9ee892544ec, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:59:01,226 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3ed23ac403772e0fe4ca9ee892544ec, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T09:59:01,226 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731751137553.f3ed23ac403772e0fe4ca9ee892544ec., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3ed23ac403772e0fe4ca9ee892544ec, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:59:01,226 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731751137553.f3ed23ac403772e0fe4ca9ee892544ec., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 because the exception is null or not the one we care about 2024-11-16T09:59:01,226 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34386 deadline: 1731751201223, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3ed23ac403772e0fe4ca9ee892544ec, server=037c58a19ddb,37267,1731751077844 2024-11-16T09:59:01,226 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3ed23ac403772e0fe4ca9ee892544ec, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T09:59:01,226 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3ed23ac403772e0fe4ca9ee892544ec, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T09:59:01,226 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731751137553.f3ed23ac403772e0fe4ca9ee892544ec., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 , the old value is region=TestAcidGuarantees,,1731751137553.f3ed23ac403772e0fe4ca9ee892544ec., hostname=037c58a19ddb,37267,1731751077844, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3ed23ac403772e0fe4ca9ee892544ec, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:59:01,226 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731751137553.f3ed23ac403772e0fe4ca9ee892544ec., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3ed23ac403772e0fe4ca9ee892544ec, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:59:01,226 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34382 deadline: 1731751201223, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3ed23ac403772e0fe4ca9ee892544ec, server=037c58a19ddb,37267,1731751077844 2024-11-16T09:59:01,226 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34408 deadline: 1731751201223, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3ed23ac403772e0fe4ca9ee892544ec, server=037c58a19ddb,37267,1731751077844 2024-11-16T09:59:01,226 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731751137553.f3ed23ac403772e0fe4ca9ee892544ec., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 because the exception is null or not the one we care about 2024-11-16T09:59:01,227 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731751137553.f3ed23ac403772e0fe4ca9ee892544ec., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 , the old value is region=TestAcidGuarantees,,1731751137553.f3ed23ac403772e0fe4ca9ee892544ec., hostname=037c58a19ddb,37267,1731751077844, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3ed23ac403772e0fe4ca9ee892544ec, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:59:01,227 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731751137553.f3ed23ac403772e0fe4ca9ee892544ec., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 , the old value is region=TestAcidGuarantees,,1731751137553.f3ed23ac403772e0fe4ca9ee892544ec., hostname=037c58a19ddb,37267,1731751077844, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3ed23ac403772e0fe4ca9ee892544ec, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:59:01,227 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731751137553.f3ed23ac403772e0fe4ca9ee892544ec., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3ed23ac403772e0fe4ca9ee892544ec, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:59:01,227 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731751137553.f3ed23ac403772e0fe4ca9ee892544ec., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3ed23ac403772e0fe4ca9ee892544ec, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:59:01,227 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731751137553.f3ed23ac403772e0fe4ca9ee892544ec., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 because the exception is null or not the one we care about 2024-11-16T09:59:01,227 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731751137553.f3ed23ac403772e0fe4ca9ee892544ec., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 because the exception is null or not the one we care about 2024-11-16T09:59:01,435 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3ed23ac403772e0fe4ca9ee892544ec, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T09:59:01,436 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34368 deadline: 1731751201433, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3ed23ac403772e0fe4ca9ee892544ec, server=037c58a19ddb,37267,1731751077844 2024-11-16T09:59:01,436 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3ed23ac403772e0fe4ca9ee892544ec, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T09:59:01,436 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731751137553.f3ed23ac403772e0fe4ca9ee892544ec., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 , the old value is region=TestAcidGuarantees,,1731751137553.f3ed23ac403772e0fe4ca9ee892544ec., hostname=037c58a19ddb,37267,1731751077844, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3ed23ac403772e0fe4ca9ee892544ec, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:59:01,436 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731751137553.f3ed23ac403772e0fe4ca9ee892544ec., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3ed23ac403772e0fe4ca9ee892544ec, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:59:01,436 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34386 deadline: 1731751201433, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3ed23ac403772e0fe4ca9ee892544ec, server=037c58a19ddb,37267,1731751077844 2024-11-16T09:59:01,436 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731751137553.f3ed23ac403772e0fe4ca9ee892544ec., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 because the exception is null or not the one we care about 2024-11-16T09:59:01,436 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731751137553.f3ed23ac403772e0fe4ca9ee892544ec., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 , the old value is region=TestAcidGuarantees,,1731751137553.f3ed23ac403772e0fe4ca9ee892544ec., hostname=037c58a19ddb,37267,1731751077844, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3ed23ac403772e0fe4ca9ee892544ec, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:59:01,436 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3ed23ac403772e0fe4ca9ee892544ec, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T09:59:01,436 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731751137553.f3ed23ac403772e0fe4ca9ee892544ec., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3ed23ac403772e0fe4ca9ee892544ec, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:59:01,436 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731751137553.f3ed23ac403772e0fe4ca9ee892544ec., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 because the exception is null or not the one we care about 2024-11-16T09:59:01,436 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34414 deadline: 1731751201433, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3ed23ac403772e0fe4ca9ee892544ec, server=037c58a19ddb,37267,1731751077844 2024-11-16T09:59:01,436 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731751137553.f3ed23ac403772e0fe4ca9ee892544ec., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 , the old value is region=TestAcidGuarantees,,1731751137553.f3ed23ac403772e0fe4ca9ee892544ec., hostname=037c58a19ddb,37267,1731751077844, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3ed23ac403772e0fe4ca9ee892544ec, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:59:01,436 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3ed23ac403772e0fe4ca9ee892544ec, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T09:59:01,436 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731751137553.f3ed23ac403772e0fe4ca9ee892544ec., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3ed23ac403772e0fe4ca9ee892544ec, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:59:01,436 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731751137553.f3ed23ac403772e0fe4ca9ee892544ec., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 because the exception is null or not the one we care about 2024-11-16T09:59:01,436 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34408 deadline: 1731751201433, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3ed23ac403772e0fe4ca9ee892544ec, server=037c58a19ddb,37267,1731751077844 2024-11-16T09:59:01,437 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3ed23ac403772e0fe4ca9ee892544ec, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T09:59:01,437 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34382 deadline: 1731751201433, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3ed23ac403772e0fe4ca9ee892544ec, server=037c58a19ddb,37267,1731751077844 2024-11-16T09:59:01,437 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731751137553.f3ed23ac403772e0fe4ca9ee892544ec., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 , the old value is region=TestAcidGuarantees,,1731751137553.f3ed23ac403772e0fe4ca9ee892544ec., hostname=037c58a19ddb,37267,1731751077844, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3ed23ac403772e0fe4ca9ee892544ec, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:59:01,437 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731751137553.f3ed23ac403772e0fe4ca9ee892544ec., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3ed23ac403772e0fe4ca9ee892544ec, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:59:01,437 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731751137553.f3ed23ac403772e0fe4ca9ee892544ec., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 because the exception is null or not the one we care about 2024-11-16T09:59:01,437 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731751137553.f3ed23ac403772e0fe4ca9ee892544ec., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 , the old value is region=TestAcidGuarantees,,1731751137553.f3ed23ac403772e0fe4ca9ee892544ec., hostname=037c58a19ddb,37267,1731751077844, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3ed23ac403772e0fe4ca9ee892544ec, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:59:01,437 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731751137553.f3ed23ac403772e0fe4ca9ee892544ec., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3ed23ac403772e0fe4ca9ee892544ec, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:59:01,437 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731751137553.f3ed23ac403772e0fe4ca9ee892544ec., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 because the exception is null or not the one we care about 2024-11-16T09:59:01,471 INFO [RS_FLUSH_OPERATIONS-regionserver/037c58a19ddb:0-0 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=50 (bloomFilter=true), to=hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/f3ed23ac403772e0fe4ca9ee892544ec/.tmp/A/3dc937ecac70408cba3ef81b89802dd1 2024-11-16T09:59:01,481 DEBUG [RS_FLUSH_OPERATIONS-regionserver/037c58a19ddb:0-0 {event_type=RS_FLUSH_REGIONS, pid=63}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/f3ed23ac403772e0fe4ca9ee892544ec/.tmp/B/edee38773ced4b9ea467a4d5655b6334 is 50, key is test_row_0/B:col10/1731751140417/Put/seqid=0 2024-11-16T09:59:01,485 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37591 is added to blk_1073742086_1262 (size=12001) 2024-11-16T09:59:01,486 INFO [RS_FLUSH_OPERATIONS-regionserver/037c58a19ddb:0-0 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=50 (bloomFilter=true), to=hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/f3ed23ac403772e0fe4ca9ee892544ec/.tmp/B/edee38773ced4b9ea467a4d5655b6334 2024-11-16T09:59:01,497 DEBUG [RS_FLUSH_OPERATIONS-regionserver/037c58a19ddb:0-0 {event_type=RS_FLUSH_REGIONS, pid=63}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/f3ed23ac403772e0fe4ca9ee892544ec/.tmp/C/b747dafa9b994f2ba6ec3cb9ada2e63d is 50, key is test_row_0/C:col10/1731751140417/Put/seqid=0 2024-11-16T09:59:01,501 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37591 is added to blk_1073742087_1263 (size=12001) 2024-11-16T09:59:01,502 INFO [RS_FLUSH_OPERATIONS-regionserver/037c58a19ddb:0-0 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=50 (bloomFilter=true), to=hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/f3ed23ac403772e0fe4ca9ee892544ec/.tmp/C/b747dafa9b994f2ba6ec3cb9ada2e63d 2024-11-16T09:59:01,507 DEBUG [RS_FLUSH_OPERATIONS-regionserver/037c58a19ddb:0-0 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/f3ed23ac403772e0fe4ca9ee892544ec/.tmp/A/3dc937ecac70408cba3ef81b89802dd1 as hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/f3ed23ac403772e0fe4ca9ee892544ec/A/3dc937ecac70408cba3ef81b89802dd1 2024-11-16T09:59:01,511 INFO [RS_FLUSH_OPERATIONS-regionserver/037c58a19ddb:0-0 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/f3ed23ac403772e0fe4ca9ee892544ec/A/3dc937ecac70408cba3ef81b89802dd1, entries=150, sequenceid=50, filesize=11.7 K 2024-11-16T09:59:01,512 DEBUG [RS_FLUSH_OPERATIONS-regionserver/037c58a19ddb:0-0 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/f3ed23ac403772e0fe4ca9ee892544ec/.tmp/B/edee38773ced4b9ea467a4d5655b6334 as hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/f3ed23ac403772e0fe4ca9ee892544ec/B/edee38773ced4b9ea467a4d5655b6334 2024-11-16T09:59:01,513 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37877 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=62 2024-11-16T09:59:01,516 INFO [RS_FLUSH_OPERATIONS-regionserver/037c58a19ddb:0-0 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/f3ed23ac403772e0fe4ca9ee892544ec/B/edee38773ced4b9ea467a4d5655b6334, entries=150, sequenceid=50, filesize=11.7 K 2024-11-16T09:59:01,518 DEBUG [RS_FLUSH_OPERATIONS-regionserver/037c58a19ddb:0-0 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/f3ed23ac403772e0fe4ca9ee892544ec/.tmp/C/b747dafa9b994f2ba6ec3cb9ada2e63d as hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/f3ed23ac403772e0fe4ca9ee892544ec/C/b747dafa9b994f2ba6ec3cb9ada2e63d 2024-11-16T09:59:01,523 INFO [RS_FLUSH_OPERATIONS-regionserver/037c58a19ddb:0-0 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/f3ed23ac403772e0fe4ca9ee892544ec/C/b747dafa9b994f2ba6ec3cb9ada2e63d, entries=150, sequenceid=50, filesize=11.7 K 2024-11-16T09:59:01,524 INFO [RS_FLUSH_OPERATIONS-regionserver/037c58a19ddb:0-0 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.HRegion(3140): Finished flush of dataSize ~67.09 KB/68700, heapSize ~176.48 KB/180720, currentSize=134.18 KB/137400 for f3ed23ac403772e0fe4ca9ee892544ec in 468ms, sequenceid=50, compaction requested=true 2024-11-16T09:59:01,524 DEBUG [RS_FLUSH_OPERATIONS-regionserver/037c58a19ddb:0-0 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.HRegion(2603): Flush status journal for f3ed23ac403772e0fe4ca9ee892544ec: 2024-11-16T09:59:01,524 DEBUG [RS_FLUSH_OPERATIONS-regionserver/037c58a19ddb:0-0 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731751137553.f3ed23ac403772e0fe4ca9ee892544ec. 2024-11-16T09:59:01,524 DEBUG [RS_FLUSH_OPERATIONS-regionserver/037c58a19ddb:0-0 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=63 2024-11-16T09:59:01,524 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37877 {}] master.HMaster(4169): Remote procedure done, pid=63 2024-11-16T09:59:01,528 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=63, resume processing ppid=62 2024-11-16T09:59:01,528 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=63, ppid=62, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 638 msec 2024-11-16T09:59:01,530 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=62, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=62, table=TestAcidGuarantees in 644 msec 2024-11-16T09:59:01,745 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] regionserver.HRegion(8855): Flush requested on f3ed23ac403772e0fe4ca9ee892544ec 2024-11-16T09:59:01,746 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-16T09:59:01,746 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing f3ed23ac403772e0fe4ca9ee892544ec 3/3 column families, dataSize=140.89 KB heapSize=369.89 KB 2024-11-16T09:59:01,752 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/f3ed23ac403772e0fe4ca9ee892544ec/.tmp/A/a4ceccf438df4ea487b2d78dc8baaad9 is 50, key is test_row_0/A:col10/1731751141108/Put/seqid=0 2024-11-16T09:59:01,757 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37591 is added to blk_1073742088_1264 (size=14341) 2024-11-16T09:59:01,780 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3ed23ac403772e0fe4ca9ee892544ec, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T09:59:01,781 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34368 deadline: 1731751201752, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3ed23ac403772e0fe4ca9ee892544ec, server=037c58a19ddb,37267,1731751077844 2024-11-16T09:59:01,781 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3ed23ac403772e0fe4ca9ee892544ec, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T09:59:01,781 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731751137553.f3ed23ac403772e0fe4ca9ee892544ec., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 , the old value is region=TestAcidGuarantees,,1731751137553.f3ed23ac403772e0fe4ca9ee892544ec., hostname=037c58a19ddb,37267,1731751077844, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3ed23ac403772e0fe4ca9ee892544ec, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:59:01,781 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731751137553.f3ed23ac403772e0fe4ca9ee892544ec., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3ed23ac403772e0fe4ca9ee892544ec, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:59:01,781 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34386 deadline: 1731751201753, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3ed23ac403772e0fe4ca9ee892544ec, server=037c58a19ddb,37267,1731751077844 2024-11-16T09:59:01,781 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731751137553.f3ed23ac403772e0fe4ca9ee892544ec., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 because the exception is null or not the one we care about 2024-11-16T09:59:01,781 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731751137553.f3ed23ac403772e0fe4ca9ee892544ec., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 , the old value is region=TestAcidGuarantees,,1731751137553.f3ed23ac403772e0fe4ca9ee892544ec., hostname=037c58a19ddb,37267,1731751077844, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3ed23ac403772e0fe4ca9ee892544ec, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:59:01,781 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731751137553.f3ed23ac403772e0fe4ca9ee892544ec., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3ed23ac403772e0fe4ca9ee892544ec, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:59:01,781 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731751137553.f3ed23ac403772e0fe4ca9ee892544ec., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 because the exception is null or not the one we care about 2024-11-16T09:59:01,785 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3ed23ac403772e0fe4ca9ee892544ec, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T09:59:01,786 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34382 deadline: 1731751201781, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3ed23ac403772e0fe4ca9ee892544ec, server=037c58a19ddb,37267,1731751077844 2024-11-16T09:59:01,786 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3ed23ac403772e0fe4ca9ee892544ec, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T09:59:01,786 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731751137553.f3ed23ac403772e0fe4ca9ee892544ec., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 , the old value is region=TestAcidGuarantees,,1731751137553.f3ed23ac403772e0fe4ca9ee892544ec., hostname=037c58a19ddb,37267,1731751077844, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3ed23ac403772e0fe4ca9ee892544ec, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:59:01,786 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731751137553.f3ed23ac403772e0fe4ca9ee892544ec., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3ed23ac403772e0fe4ca9ee892544ec, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:59:01,786 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34408 deadline: 1731751201781, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3ed23ac403772e0fe4ca9ee892544ec, server=037c58a19ddb,37267,1731751077844 2024-11-16T09:59:01,786 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731751137553.f3ed23ac403772e0fe4ca9ee892544ec., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 because the exception is null or not the one we care about 2024-11-16T09:59:01,786 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731751137553.f3ed23ac403772e0fe4ca9ee892544ec., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 , the old value is region=TestAcidGuarantees,,1731751137553.f3ed23ac403772e0fe4ca9ee892544ec., hostname=037c58a19ddb,37267,1731751077844, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3ed23ac403772e0fe4ca9ee892544ec, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:59:01,786 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3ed23ac403772e0fe4ca9ee892544ec, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T09:59:01,786 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731751137553.f3ed23ac403772e0fe4ca9ee892544ec., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3ed23ac403772e0fe4ca9ee892544ec, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:59:01,786 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731751137553.f3ed23ac403772e0fe4ca9ee892544ec., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 because the exception is null or not the one we care about 2024-11-16T09:59:01,786 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34414 deadline: 1731751201782, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3ed23ac403772e0fe4ca9ee892544ec, server=037c58a19ddb,37267,1731751077844 2024-11-16T09:59:01,786 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731751137553.f3ed23ac403772e0fe4ca9ee892544ec., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 , the old value is region=TestAcidGuarantees,,1731751137553.f3ed23ac403772e0fe4ca9ee892544ec., hostname=037c58a19ddb,37267,1731751077844, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3ed23ac403772e0fe4ca9ee892544ec, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:59:01,787 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731751137553.f3ed23ac403772e0fe4ca9ee892544ec., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3ed23ac403772e0fe4ca9ee892544ec, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:59:01,787 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731751137553.f3ed23ac403772e0fe4ca9ee892544ec., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 because the exception is null or not the one we care about 2024-11-16T09:59:01,884 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3ed23ac403772e0fe4ca9ee892544ec, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T09:59:01,884 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34368 deadline: 1731751201883, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3ed23ac403772e0fe4ca9ee892544ec, server=037c58a19ddb,37267,1731751077844 2024-11-16T09:59:01,884 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3ed23ac403772e0fe4ca9ee892544ec, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T09:59:01,884 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731751137553.f3ed23ac403772e0fe4ca9ee892544ec., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 , the old value is region=TestAcidGuarantees,,1731751137553.f3ed23ac403772e0fe4ca9ee892544ec., hostname=037c58a19ddb,37267,1731751077844, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3ed23ac403772e0fe4ca9ee892544ec, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:59:01,884 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731751137553.f3ed23ac403772e0fe4ca9ee892544ec., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3ed23ac403772e0fe4ca9ee892544ec, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:59:01,884 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731751137553.f3ed23ac403772e0fe4ca9ee892544ec., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 because the exception is null or not the one we care about 2024-11-16T09:59:01,885 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34386 deadline: 1731751201883, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3ed23ac403772e0fe4ca9ee892544ec, server=037c58a19ddb,37267,1731751077844 2024-11-16T09:59:01,885 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731751137553.f3ed23ac403772e0fe4ca9ee892544ec., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 , the old value is region=TestAcidGuarantees,,1731751137553.f3ed23ac403772e0fe4ca9ee892544ec., hostname=037c58a19ddb,37267,1731751077844, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3ed23ac403772e0fe4ca9ee892544ec, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:59:01,885 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731751137553.f3ed23ac403772e0fe4ca9ee892544ec., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3ed23ac403772e0fe4ca9ee892544ec, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:59:01,885 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731751137553.f3ed23ac403772e0fe4ca9ee892544ec., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 because the exception is null or not the one we care about 2024-11-16T09:59:01,896 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3ed23ac403772e0fe4ca9ee892544ec, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T09:59:01,896 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34382 deadline: 1731751201893, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3ed23ac403772e0fe4ca9ee892544ec, server=037c58a19ddb,37267,1731751077844 2024-11-16T09:59:01,896 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731751137553.f3ed23ac403772e0fe4ca9ee892544ec., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 , the old value is region=TestAcidGuarantees,,1731751137553.f3ed23ac403772e0fe4ca9ee892544ec., hostname=037c58a19ddb,37267,1731751077844, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3ed23ac403772e0fe4ca9ee892544ec, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:59:01,896 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3ed23ac403772e0fe4ca9ee892544ec, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T09:59:01,896 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731751137553.f3ed23ac403772e0fe4ca9ee892544ec., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3ed23ac403772e0fe4ca9ee892544ec, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:59:01,896 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731751137553.f3ed23ac403772e0fe4ca9ee892544ec., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 because the exception is null or not the one we care about 2024-11-16T09:59:01,896 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34408 deadline: 1731751201893, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3ed23ac403772e0fe4ca9ee892544ec, server=037c58a19ddb,37267,1731751077844 2024-11-16T09:59:01,896 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731751137553.f3ed23ac403772e0fe4ca9ee892544ec., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 , the old value is region=TestAcidGuarantees,,1731751137553.f3ed23ac403772e0fe4ca9ee892544ec., hostname=037c58a19ddb,37267,1731751077844, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3ed23ac403772e0fe4ca9ee892544ec, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:59:01,896 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3ed23ac403772e0fe4ca9ee892544ec, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T09:59:01,896 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731751137553.f3ed23ac403772e0fe4ca9ee892544ec., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3ed23ac403772e0fe4ca9ee892544ec, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:59:01,896 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731751137553.f3ed23ac403772e0fe4ca9ee892544ec., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 because the exception is null or not the one we care about 2024-11-16T09:59:01,897 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34414 deadline: 1731751201893, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3ed23ac403772e0fe4ca9ee892544ec, server=037c58a19ddb,37267,1731751077844 2024-11-16T09:59:01,897 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731751137553.f3ed23ac403772e0fe4ca9ee892544ec., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 , the old value is region=TestAcidGuarantees,,1731751137553.f3ed23ac403772e0fe4ca9ee892544ec., hostname=037c58a19ddb,37267,1731751077844, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3ed23ac403772e0fe4ca9ee892544ec, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:59:01,897 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731751137553.f3ed23ac403772e0fe4ca9ee892544ec., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3ed23ac403772e0fe4ca9ee892544ec, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:59:01,897 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731751137553.f3ed23ac403772e0fe4ca9ee892544ec., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 because the exception is null or not the one we care about 2024-11-16T09:59:02,022 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37877 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=62 2024-11-16T09:59:02,023 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-16T09:59:02,023 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37877 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-16T09:59:02,024 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37877 {}] procedure2.ProcedureExecutor(1139): Stored pid=64, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=64, table=TestAcidGuarantees 2024-11-16T09:59:02,025 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37877 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=64 2024-11-16T09:59:02,025 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=64, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=64, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-16T09:59:02,026 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=64, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=64, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-16T09:59:02,026 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=65, ppid=64, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-16T09:59:02,096 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3ed23ac403772e0fe4ca9ee892544ec, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T09:59:02,096 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34368 deadline: 1731751202093, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3ed23ac403772e0fe4ca9ee892544ec, server=037c58a19ddb,37267,1731751077844 2024-11-16T09:59:02,096 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3ed23ac403772e0fe4ca9ee892544ec, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T09:59:02,096 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34386 deadline: 1731751202093, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3ed23ac403772e0fe4ca9ee892544ec, server=037c58a19ddb,37267,1731751077844 2024-11-16T09:59:02,097 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731751137553.f3ed23ac403772e0fe4ca9ee892544ec., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 , the old value is region=TestAcidGuarantees,,1731751137553.f3ed23ac403772e0fe4ca9ee892544ec., hostname=037c58a19ddb,37267,1731751077844, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3ed23ac403772e0fe4ca9ee892544ec, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:59:02,097 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731751137553.f3ed23ac403772e0fe4ca9ee892544ec., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3ed23ac403772e0fe4ca9ee892544ec, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:59:02,097 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731751137553.f3ed23ac403772e0fe4ca9ee892544ec., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 because the exception is null or not the one we care about 2024-11-16T09:59:02,097 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731751137553.f3ed23ac403772e0fe4ca9ee892544ec., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 , the old value is region=TestAcidGuarantees,,1731751137553.f3ed23ac403772e0fe4ca9ee892544ec., hostname=037c58a19ddb,37267,1731751077844, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3ed23ac403772e0fe4ca9ee892544ec, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:59:02,097 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731751137553.f3ed23ac403772e0fe4ca9ee892544ec., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3ed23ac403772e0fe4ca9ee892544ec, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:59:02,097 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731751137553.f3ed23ac403772e0fe4ca9ee892544ec., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 because the exception is null or not the one we care about 2024-11-16T09:59:02,104 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3ed23ac403772e0fe4ca9ee892544ec, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T09:59:02,104 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34408 deadline: 1731751202103, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3ed23ac403772e0fe4ca9ee892544ec, server=037c58a19ddb,37267,1731751077844 2024-11-16T09:59:02,105 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3ed23ac403772e0fe4ca9ee892544ec, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T09:59:02,105 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34382 deadline: 1731751202103, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3ed23ac403772e0fe4ca9ee892544ec, server=037c58a19ddb,37267,1731751077844 2024-11-16T09:59:02,105 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731751137553.f3ed23ac403772e0fe4ca9ee892544ec., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 , the old value is region=TestAcidGuarantees,,1731751137553.f3ed23ac403772e0fe4ca9ee892544ec., hostname=037c58a19ddb,37267,1731751077844, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3ed23ac403772e0fe4ca9ee892544ec, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:59:02,105 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731751137553.f3ed23ac403772e0fe4ca9ee892544ec., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3ed23ac403772e0fe4ca9ee892544ec, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:59:02,105 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731751137553.f3ed23ac403772e0fe4ca9ee892544ec., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 because the exception is null or not the one we care about 2024-11-16T09:59:02,105 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731751137553.f3ed23ac403772e0fe4ca9ee892544ec., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 , the old value is region=TestAcidGuarantees,,1731751137553.f3ed23ac403772e0fe4ca9ee892544ec., hostname=037c58a19ddb,37267,1731751077844, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3ed23ac403772e0fe4ca9ee892544ec, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:59:02,105 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3ed23ac403772e0fe4ca9ee892544ec, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T09:59:02,105 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731751137553.f3ed23ac403772e0fe4ca9ee892544ec., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3ed23ac403772e0fe4ca9ee892544ec, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:59:02,105 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34414 deadline: 1731751202103, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3ed23ac403772e0fe4ca9ee892544ec, server=037c58a19ddb,37267,1731751077844 2024-11-16T09:59:02,105 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731751137553.f3ed23ac403772e0fe4ca9ee892544ec., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 because the exception is null or not the one we care about 2024-11-16T09:59:02,105 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731751137553.f3ed23ac403772e0fe4ca9ee892544ec., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 , the old value is region=TestAcidGuarantees,,1731751137553.f3ed23ac403772e0fe4ca9ee892544ec., hostname=037c58a19ddb,37267,1731751077844, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3ed23ac403772e0fe4ca9ee892544ec, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:59:02,105 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731751137553.f3ed23ac403772e0fe4ca9ee892544ec., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3ed23ac403772e0fe4ca9ee892544ec, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:59:02,105 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731751137553.f3ed23ac403772e0fe4ca9ee892544ec., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 because the exception is null or not the one we care about 2024-11-16T09:59:02,133 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37877 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=64 2024-11-16T09:59:02,158 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=74 (bloomFilter=true), to=hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/f3ed23ac403772e0fe4ca9ee892544ec/.tmp/A/a4ceccf438df4ea487b2d78dc8baaad9 2024-11-16T09:59:02,167 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/f3ed23ac403772e0fe4ca9ee892544ec/.tmp/B/7b8eb25c363c4c66a4eb1bc0ba18a151 is 50, key is test_row_0/B:col10/1731751141108/Put/seqid=0 2024-11-16T09:59:02,178 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37591 is added to blk_1073742089_1265 (size=12001) 2024-11-16T09:59:02,179 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=37267 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=65 2024-11-16T09:59:02,179 DEBUG [RS_FLUSH_OPERATIONS-regionserver/037c58a19ddb:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731751137553.f3ed23ac403772e0fe4ca9ee892544ec. 2024-11-16T09:59:02,179 DEBUG [RS_FLUSH_OPERATIONS-regionserver/037c58a19ddb:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731751137553.f3ed23ac403772e0fe4ca9ee892544ec. as already flushing 2024-11-16T09:59:02,179 DEBUG [RS_FLUSH_OPERATIONS-regionserver/037c58a19ddb:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731751137553.f3ed23ac403772e0fe4ca9ee892544ec. 2024-11-16T09:59:02,179 ERROR [RS_FLUSH_OPERATIONS-regionserver/037c58a19ddb:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] handler.RSProcedureHandler(58): pid=65 java.io.IOException: Unable to complete flush {ENCODED => f3ed23ac403772e0fe4ca9ee892544ec, NAME => 'TestAcidGuarantees,,1731751137553.f3ed23ac403772e0fe4ca9ee892544ec.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T09:59:02,179 DEBUG [RS_FLUSH_OPERATIONS-regionserver/037c58a19ddb:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=65 java.io.IOException: Unable to complete flush {ENCODED => f3ed23ac403772e0fe4ca9ee892544ec, NAME => 'TestAcidGuarantees,,1731751137553.f3ed23ac403772e0fe4ca9ee892544ec.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T09:59:02,180 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37877 {}] master.HMaster(4177): Remote procedure failed, pid=65 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => f3ed23ac403772e0fe4ca9ee892544ec, NAME => 'TestAcidGuarantees,,1731751137553.f3ed23ac403772e0fe4ca9ee892544ec.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => f3ed23ac403772e0fe4ca9ee892544ec, NAME => 'TestAcidGuarantees,,1731751137553.f3ed23ac403772e0fe4ca9ee892544ec.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T09:59:02,331 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=37267 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=65 2024-11-16T09:59:02,331 DEBUG [RS_FLUSH_OPERATIONS-regionserver/037c58a19ddb:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731751137553.f3ed23ac403772e0fe4ca9ee892544ec. 2024-11-16T09:59:02,332 DEBUG [RS_FLUSH_OPERATIONS-regionserver/037c58a19ddb:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731751137553.f3ed23ac403772e0fe4ca9ee892544ec. as already flushing 2024-11-16T09:59:02,332 DEBUG [RS_FLUSH_OPERATIONS-regionserver/037c58a19ddb:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731751137553.f3ed23ac403772e0fe4ca9ee892544ec. 2024-11-16T09:59:02,332 ERROR [RS_FLUSH_OPERATIONS-regionserver/037c58a19ddb:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] handler.RSProcedureHandler(58): pid=65 java.io.IOException: Unable to complete flush {ENCODED => f3ed23ac403772e0fe4ca9ee892544ec, NAME => 'TestAcidGuarantees,,1731751137553.f3ed23ac403772e0fe4ca9ee892544ec.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T09:59:02,332 DEBUG [RS_FLUSH_OPERATIONS-regionserver/037c58a19ddb:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=65 java.io.IOException: Unable to complete flush {ENCODED => f3ed23ac403772e0fe4ca9ee892544ec, NAME => 'TestAcidGuarantees,,1731751137553.f3ed23ac403772e0fe4ca9ee892544ec.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T09:59:02,333 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37877 {}] master.HMaster(4177): Remote procedure failed, pid=65 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => f3ed23ac403772e0fe4ca9ee892544ec, NAME => 'TestAcidGuarantees,,1731751137553.f3ed23ac403772e0fe4ca9ee892544ec.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => f3ed23ac403772e0fe4ca9ee892544ec, NAME => 'TestAcidGuarantees,,1731751137553.f3ed23ac403772e0fe4ca9ee892544ec.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T09:59:02,343 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37877 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=64 2024-11-16T09:59:02,404 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3ed23ac403772e0fe4ca9ee892544ec, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T09:59:02,404 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34368 deadline: 1731751202403, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3ed23ac403772e0fe4ca9ee892544ec, server=037c58a19ddb,37267,1731751077844 2024-11-16T09:59:02,405 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731751137553.f3ed23ac403772e0fe4ca9ee892544ec., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 , the old value is region=TestAcidGuarantees,,1731751137553.f3ed23ac403772e0fe4ca9ee892544ec., hostname=037c58a19ddb,37267,1731751077844, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3ed23ac403772e0fe4ca9ee892544ec, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:59:02,405 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731751137553.f3ed23ac403772e0fe4ca9ee892544ec., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3ed23ac403772e0fe4ca9ee892544ec, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:59:02,405 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3ed23ac403772e0fe4ca9ee892544ec, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T09:59:02,405 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731751137553.f3ed23ac403772e0fe4ca9ee892544ec., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 because the exception is null or not the one we care about 2024-11-16T09:59:02,405 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34386 deadline: 1731751202403, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3ed23ac403772e0fe4ca9ee892544ec, server=037c58a19ddb,37267,1731751077844 2024-11-16T09:59:02,405 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731751137553.f3ed23ac403772e0fe4ca9ee892544ec., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 , the old value is region=TestAcidGuarantees,,1731751137553.f3ed23ac403772e0fe4ca9ee892544ec., hostname=037c58a19ddb,37267,1731751077844, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3ed23ac403772e0fe4ca9ee892544ec, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:59:02,405 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731751137553.f3ed23ac403772e0fe4ca9ee892544ec., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3ed23ac403772e0fe4ca9ee892544ec, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:59:02,405 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731751137553.f3ed23ac403772e0fe4ca9ee892544ec., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 because the exception is null or not the one we care about 2024-11-16T09:59:02,415 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3ed23ac403772e0fe4ca9ee892544ec, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T09:59:02,415 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34408 deadline: 1731751202413, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3ed23ac403772e0fe4ca9ee892544ec, server=037c58a19ddb,37267,1731751077844 2024-11-16T09:59:02,415 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731751137553.f3ed23ac403772e0fe4ca9ee892544ec., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 , the old value is region=TestAcidGuarantees,,1731751137553.f3ed23ac403772e0fe4ca9ee892544ec., hostname=037c58a19ddb,37267,1731751077844, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3ed23ac403772e0fe4ca9ee892544ec, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:59:02,415 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731751137553.f3ed23ac403772e0fe4ca9ee892544ec., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3ed23ac403772e0fe4ca9ee892544ec, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:59:02,416 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731751137553.f3ed23ac403772e0fe4ca9ee892544ec., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 because the exception is null or not the one we care about 2024-11-16T09:59:02,416 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3ed23ac403772e0fe4ca9ee892544ec, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T09:59:02,416 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34382 deadline: 1731751202413, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3ed23ac403772e0fe4ca9ee892544ec, server=037c58a19ddb,37267,1731751077844 2024-11-16T09:59:02,417 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3ed23ac403772e0fe4ca9ee892544ec, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T09:59:02,417 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731751137553.f3ed23ac403772e0fe4ca9ee892544ec., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 , the old value is region=TestAcidGuarantees,,1731751137553.f3ed23ac403772e0fe4ca9ee892544ec., hostname=037c58a19ddb,37267,1731751077844, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3ed23ac403772e0fe4ca9ee892544ec, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:59:02,417 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34414 deadline: 1731751202413, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3ed23ac403772e0fe4ca9ee892544ec, server=037c58a19ddb,37267,1731751077844 2024-11-16T09:59:02,417 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731751137553.f3ed23ac403772e0fe4ca9ee892544ec., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3ed23ac403772e0fe4ca9ee892544ec, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:59:02,417 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731751137553.f3ed23ac403772e0fe4ca9ee892544ec., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 because the exception is null or not the one we care about 2024-11-16T09:59:02,417 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731751137553.f3ed23ac403772e0fe4ca9ee892544ec., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 , the old value is region=TestAcidGuarantees,,1731751137553.f3ed23ac403772e0fe4ca9ee892544ec., hostname=037c58a19ddb,37267,1731751077844, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3ed23ac403772e0fe4ca9ee892544ec, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:59:02,417 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731751137553.f3ed23ac403772e0fe4ca9ee892544ec., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3ed23ac403772e0fe4ca9ee892544ec, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:59:02,417 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731751137553.f3ed23ac403772e0fe4ca9ee892544ec., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 because the exception is null or not the one we care about 2024-11-16T09:59:02,485 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=37267 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=65 2024-11-16T09:59:02,485 DEBUG [RS_FLUSH_OPERATIONS-regionserver/037c58a19ddb:0-0 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731751137553.f3ed23ac403772e0fe4ca9ee892544ec. 2024-11-16T09:59:02,485 DEBUG [RS_FLUSH_OPERATIONS-regionserver/037c58a19ddb:0-0 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731751137553.f3ed23ac403772e0fe4ca9ee892544ec. as already flushing 2024-11-16T09:59:02,485 DEBUG [RS_FLUSH_OPERATIONS-regionserver/037c58a19ddb:0-0 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731751137553.f3ed23ac403772e0fe4ca9ee892544ec. 2024-11-16T09:59:02,485 ERROR [RS_FLUSH_OPERATIONS-regionserver/037c58a19ddb:0-0 {event_type=RS_FLUSH_REGIONS, pid=65}] handler.RSProcedureHandler(58): pid=65 java.io.IOException: Unable to complete flush {ENCODED => f3ed23ac403772e0fe4ca9ee892544ec, NAME => 'TestAcidGuarantees,,1731751137553.f3ed23ac403772e0fe4ca9ee892544ec.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T09:59:02,485 DEBUG [RS_FLUSH_OPERATIONS-regionserver/037c58a19ddb:0-0 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=65 java.io.IOException: Unable to complete flush {ENCODED => f3ed23ac403772e0fe4ca9ee892544ec, NAME => 'TestAcidGuarantees,,1731751137553.f3ed23ac403772e0fe4ca9ee892544ec.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T09:59:02,486 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37877 {}] master.HMaster(4177): Remote procedure failed, pid=65 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => f3ed23ac403772e0fe4ca9ee892544ec, NAME => 'TestAcidGuarantees,,1731751137553.f3ed23ac403772e0fe4ca9ee892544ec.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => f3ed23ac403772e0fe4ca9ee892544ec, NAME => 'TestAcidGuarantees,,1731751137553.f3ed23ac403772e0fe4ca9ee892544ec.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T09:59:02,579 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=74 (bloomFilter=true), to=hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/f3ed23ac403772e0fe4ca9ee892544ec/.tmp/B/7b8eb25c363c4c66a4eb1bc0ba18a151 2024-11-16T09:59:02,586 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/f3ed23ac403772e0fe4ca9ee892544ec/.tmp/C/b20aebb0d817468f8d6fb345deac0dc9 is 50, key is test_row_0/C:col10/1731751141108/Put/seqid=0 2024-11-16T09:59:02,592 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37591 is added to blk_1073742090_1266 (size=12001) 2024-11-16T09:59:02,638 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=37267 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=65 2024-11-16T09:59:02,638 DEBUG [RS_FLUSH_OPERATIONS-regionserver/037c58a19ddb:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731751137553.f3ed23ac403772e0fe4ca9ee892544ec. 2024-11-16T09:59:02,638 DEBUG [RS_FLUSH_OPERATIONS-regionserver/037c58a19ddb:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731751137553.f3ed23ac403772e0fe4ca9ee892544ec. as already flushing 2024-11-16T09:59:02,638 DEBUG [RS_FLUSH_OPERATIONS-regionserver/037c58a19ddb:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731751137553.f3ed23ac403772e0fe4ca9ee892544ec. 2024-11-16T09:59:02,639 ERROR [RS_FLUSH_OPERATIONS-regionserver/037c58a19ddb:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] handler.RSProcedureHandler(58): pid=65 java.io.IOException: Unable to complete flush {ENCODED => f3ed23ac403772e0fe4ca9ee892544ec, NAME => 'TestAcidGuarantees,,1731751137553.f3ed23ac403772e0fe4ca9ee892544ec.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T09:59:02,639 DEBUG [RS_FLUSH_OPERATIONS-regionserver/037c58a19ddb:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=65 java.io.IOException: Unable to complete flush {ENCODED => f3ed23ac403772e0fe4ca9ee892544ec, NAME => 'TestAcidGuarantees,,1731751137553.f3ed23ac403772e0fe4ca9ee892544ec.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T09:59:02,639 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37877 {}] master.HMaster(4177): Remote procedure failed, pid=65 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => f3ed23ac403772e0fe4ca9ee892544ec, NAME => 'TestAcidGuarantees,,1731751137553.f3ed23ac403772e0fe4ca9ee892544ec.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => f3ed23ac403772e0fe4ca9ee892544ec, NAME => 'TestAcidGuarantees,,1731751137553.f3ed23ac403772e0fe4ca9ee892544ec.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T09:59:02,653 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37877 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=64 2024-11-16T09:59:02,700 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-11-16T09:59:02,790 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=37267 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=65 2024-11-16T09:59:02,791 DEBUG [RS_FLUSH_OPERATIONS-regionserver/037c58a19ddb:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731751137553.f3ed23ac403772e0fe4ca9ee892544ec. 2024-11-16T09:59:02,791 DEBUG [RS_FLUSH_OPERATIONS-regionserver/037c58a19ddb:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731751137553.f3ed23ac403772e0fe4ca9ee892544ec. as already flushing 2024-11-16T09:59:02,791 DEBUG [RS_FLUSH_OPERATIONS-regionserver/037c58a19ddb:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731751137553.f3ed23ac403772e0fe4ca9ee892544ec. 2024-11-16T09:59:02,791 ERROR [RS_FLUSH_OPERATIONS-regionserver/037c58a19ddb:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] handler.RSProcedureHandler(58): pid=65 java.io.IOException: Unable to complete flush {ENCODED => f3ed23ac403772e0fe4ca9ee892544ec, NAME => 'TestAcidGuarantees,,1731751137553.f3ed23ac403772e0fe4ca9ee892544ec.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T09:59:02,791 DEBUG [RS_FLUSH_OPERATIONS-regionserver/037c58a19ddb:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=65 java.io.IOException: Unable to complete flush {ENCODED => f3ed23ac403772e0fe4ca9ee892544ec, NAME => 'TestAcidGuarantees,,1731751137553.f3ed23ac403772e0fe4ca9ee892544ec.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T09:59:02,792 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37877 {}] master.HMaster(4177): Remote procedure failed, pid=65 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => f3ed23ac403772e0fe4ca9ee892544ec, NAME => 'TestAcidGuarantees,,1731751137553.f3ed23ac403772e0fe4ca9ee892544ec.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => f3ed23ac403772e0fe4ca9ee892544ec, NAME => 'TestAcidGuarantees,,1731751137553.f3ed23ac403772e0fe4ca9ee892544ec.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T09:59:02,915 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3ed23ac403772e0fe4ca9ee892544ec, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T09:59:02,915 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34368 deadline: 1731751202913, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3ed23ac403772e0fe4ca9ee892544ec, server=037c58a19ddb,37267,1731751077844 2024-11-16T09:59:02,916 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3ed23ac403772e0fe4ca9ee892544ec, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T09:59:02,916 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731751137553.f3ed23ac403772e0fe4ca9ee892544ec., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 , the old value is region=TestAcidGuarantees,,1731751137553.f3ed23ac403772e0fe4ca9ee892544ec., hostname=037c58a19ddb,37267,1731751077844, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3ed23ac403772e0fe4ca9ee892544ec, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:59:02,916 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731751137553.f3ed23ac403772e0fe4ca9ee892544ec., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3ed23ac403772e0fe4ca9ee892544ec, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:59:02,916 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731751137553.f3ed23ac403772e0fe4ca9ee892544ec., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 because the exception is null or not the one we care about 2024-11-16T09:59:02,916 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34386 deadline: 1731751202913, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3ed23ac403772e0fe4ca9ee892544ec, server=037c58a19ddb,37267,1731751077844 2024-11-16T09:59:02,916 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731751137553.f3ed23ac403772e0fe4ca9ee892544ec., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 , the old value is region=TestAcidGuarantees,,1731751137553.f3ed23ac403772e0fe4ca9ee892544ec., hostname=037c58a19ddb,37267,1731751077844, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3ed23ac403772e0fe4ca9ee892544ec, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:59:02,916 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731751137553.f3ed23ac403772e0fe4ca9ee892544ec., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3ed23ac403772e0fe4ca9ee892544ec, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:59:02,916 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731751137553.f3ed23ac403772e0fe4ca9ee892544ec., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 because the exception is null or not the one we care about 2024-11-16T09:59:02,924 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3ed23ac403772e0fe4ca9ee892544ec, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T09:59:02,924 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34408 deadline: 1731751202922, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3ed23ac403772e0fe4ca9ee892544ec, server=037c58a19ddb,37267,1731751077844 2024-11-16T09:59:02,924 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3ed23ac403772e0fe4ca9ee892544ec, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T09:59:02,924 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731751137553.f3ed23ac403772e0fe4ca9ee892544ec., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 , the old value is region=TestAcidGuarantees,,1731751137553.f3ed23ac403772e0fe4ca9ee892544ec., hostname=037c58a19ddb,37267,1731751077844, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3ed23ac403772e0fe4ca9ee892544ec, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:59:02,924 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731751137553.f3ed23ac403772e0fe4ca9ee892544ec., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3ed23ac403772e0fe4ca9ee892544ec, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:59:02,924 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731751137553.f3ed23ac403772e0fe4ca9ee892544ec., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 because the exception is null or not the one we care about 2024-11-16T09:59:02,924 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34382 deadline: 1731751202923, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3ed23ac403772e0fe4ca9ee892544ec, server=037c58a19ddb,37267,1731751077844 2024-11-16T09:59:02,925 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3ed23ac403772e0fe4ca9ee892544ec, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T09:59:02,925 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731751137553.f3ed23ac403772e0fe4ca9ee892544ec., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 , the old value is region=TestAcidGuarantees,,1731751137553.f3ed23ac403772e0fe4ca9ee892544ec., hostname=037c58a19ddb,37267,1731751077844, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3ed23ac403772e0fe4ca9ee892544ec, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:59:02,925 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34414 deadline: 1731751202923, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3ed23ac403772e0fe4ca9ee892544ec, server=037c58a19ddb,37267,1731751077844 2024-11-16T09:59:02,925 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731751137553.f3ed23ac403772e0fe4ca9ee892544ec., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3ed23ac403772e0fe4ca9ee892544ec, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:59:02,925 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731751137553.f3ed23ac403772e0fe4ca9ee892544ec., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 because the exception is null or not the one we care about 2024-11-16T09:59:02,925 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731751137553.f3ed23ac403772e0fe4ca9ee892544ec., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 , the old value is region=TestAcidGuarantees,,1731751137553.f3ed23ac403772e0fe4ca9ee892544ec., hostname=037c58a19ddb,37267,1731751077844, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3ed23ac403772e0fe4ca9ee892544ec, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:59:02,925 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731751137553.f3ed23ac403772e0fe4ca9ee892544ec., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3ed23ac403772e0fe4ca9ee892544ec, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:59:02,925 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731751137553.f3ed23ac403772e0fe4ca9ee892544ec., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 because the exception is null or not the one we care about 2024-11-16T09:59:02,944 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=37267 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=65 2024-11-16T09:59:02,944 DEBUG [RS_FLUSH_OPERATIONS-regionserver/037c58a19ddb:0-0 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731751137553.f3ed23ac403772e0fe4ca9ee892544ec. 2024-11-16T09:59:02,944 DEBUG [RS_FLUSH_OPERATIONS-regionserver/037c58a19ddb:0-0 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731751137553.f3ed23ac403772e0fe4ca9ee892544ec. as already flushing 2024-11-16T09:59:02,944 DEBUG [RS_FLUSH_OPERATIONS-regionserver/037c58a19ddb:0-0 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731751137553.f3ed23ac403772e0fe4ca9ee892544ec. 2024-11-16T09:59:02,944 ERROR [RS_FLUSH_OPERATIONS-regionserver/037c58a19ddb:0-0 {event_type=RS_FLUSH_REGIONS, pid=65}] handler.RSProcedureHandler(58): pid=65 java.io.IOException: Unable to complete flush {ENCODED => f3ed23ac403772e0fe4ca9ee892544ec, NAME => 'TestAcidGuarantees,,1731751137553.f3ed23ac403772e0fe4ca9ee892544ec.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T09:59:02,945 DEBUG [RS_FLUSH_OPERATIONS-regionserver/037c58a19ddb:0-0 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=65 java.io.IOException: Unable to complete flush {ENCODED => f3ed23ac403772e0fe4ca9ee892544ec, NAME => 'TestAcidGuarantees,,1731751137553.f3ed23ac403772e0fe4ca9ee892544ec.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T09:59:02,945 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37877 {}] master.HMaster(4177): Remote procedure failed, pid=65 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => f3ed23ac403772e0fe4ca9ee892544ec, NAME => 'TestAcidGuarantees,,1731751137553.f3ed23ac403772e0fe4ca9ee892544ec.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => f3ed23ac403772e0fe4ca9ee892544ec, NAME => 'TestAcidGuarantees,,1731751137553.f3ed23ac403772e0fe4ca9ee892544ec.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T09:59:02,992 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=74 (bloomFilter=true), to=hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/f3ed23ac403772e0fe4ca9ee892544ec/.tmp/C/b20aebb0d817468f8d6fb345deac0dc9 2024-11-16T09:59:02,997 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/f3ed23ac403772e0fe4ca9ee892544ec/.tmp/A/a4ceccf438df4ea487b2d78dc8baaad9 as hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/f3ed23ac403772e0fe4ca9ee892544ec/A/a4ceccf438df4ea487b2d78dc8baaad9 2024-11-16T09:59:03,001 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/f3ed23ac403772e0fe4ca9ee892544ec/A/a4ceccf438df4ea487b2d78dc8baaad9, entries=200, sequenceid=74, filesize=14.0 K 2024-11-16T09:59:03,003 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/f3ed23ac403772e0fe4ca9ee892544ec/.tmp/B/7b8eb25c363c4c66a4eb1bc0ba18a151 as hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/f3ed23ac403772e0fe4ca9ee892544ec/B/7b8eb25c363c4c66a4eb1bc0ba18a151 2024-11-16T09:59:03,008 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/f3ed23ac403772e0fe4ca9ee892544ec/B/7b8eb25c363c4c66a4eb1bc0ba18a151, entries=150, sequenceid=74, filesize=11.7 K 2024-11-16T09:59:03,009 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/f3ed23ac403772e0fe4ca9ee892544ec/.tmp/C/b20aebb0d817468f8d6fb345deac0dc9 as hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/f3ed23ac403772e0fe4ca9ee892544ec/C/b20aebb0d817468f8d6fb345deac0dc9 2024-11-16T09:59:03,012 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/f3ed23ac403772e0fe4ca9ee892544ec/C/b20aebb0d817468f8d6fb345deac0dc9, entries=150, sequenceid=74, filesize=11.7 K 2024-11-16T09:59:03,013 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~140.89 KB/144270, heapSize ~369.84 KB/378720, currentSize=60.38 KB/61830 for f3ed23ac403772e0fe4ca9ee892544ec in 1267ms, sequenceid=74, compaction requested=true 2024-11-16T09:59:03,013 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for f3ed23ac403772e0fe4ca9ee892544ec: 2024-11-16T09:59:03,013 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store f3ed23ac403772e0fe4ca9ee892544ec:A, priority=-2147483648, current under compaction store size is 1 2024-11-16T09:59:03,013 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-16T09:59:03,014 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store f3ed23ac403772e0fe4ca9ee892544ec:B, priority=-2147483648, current under compaction store size is 2 2024-11-16T09:59:03,014 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-16T09:59:03,014 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store f3ed23ac403772e0fe4ca9ee892544ec:C, priority=-2147483648, current under compaction store size is 3 2024-11-16T09:59:03,014 DEBUG [RS:0;037c58a19ddb:37267-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-16T09:59:03,014 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-16T09:59:03,014 DEBUG [RS:0;037c58a19ddb:37267-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-16T09:59:03,015 DEBUG [RS:0;037c58a19ddb:37267-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 50344 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-16T09:59:03,015 DEBUG [RS:0;037c58a19ddb:37267-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48004 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-16T09:59:03,015 DEBUG [RS:0;037c58a19ddb:37267-shortCompactions-0 {}] regionserver.HStore(1541): f3ed23ac403772e0fe4ca9ee892544ec/A is initiating minor compaction (all files) 2024-11-16T09:59:03,015 DEBUG [RS:0;037c58a19ddb:37267-longCompactions-0 {}] regionserver.HStore(1541): f3ed23ac403772e0fe4ca9ee892544ec/B is initiating minor compaction (all files) 2024-11-16T09:59:03,015 INFO [RS:0;037c58a19ddb:37267-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of f3ed23ac403772e0fe4ca9ee892544ec/A in TestAcidGuarantees,,1731751137553.f3ed23ac403772e0fe4ca9ee892544ec. 2024-11-16T09:59:03,015 INFO [RS:0;037c58a19ddb:37267-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of f3ed23ac403772e0fe4ca9ee892544ec/B in TestAcidGuarantees,,1731751137553.f3ed23ac403772e0fe4ca9ee892544ec. 2024-11-16T09:59:03,015 INFO [RS:0;037c58a19ddb:37267-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/f3ed23ac403772e0fe4ca9ee892544ec/A/c97cbf9e65af4cc4afa063288dd53d11, hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/f3ed23ac403772e0fe4ca9ee892544ec/A/c3e74913af05458d9452010d8e8044c0, hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/f3ed23ac403772e0fe4ca9ee892544ec/A/3dc937ecac70408cba3ef81b89802dd1, hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/f3ed23ac403772e0fe4ca9ee892544ec/A/a4ceccf438df4ea487b2d78dc8baaad9] into tmpdir=hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/f3ed23ac403772e0fe4ca9ee892544ec/.tmp, totalSize=49.2 K 2024-11-16T09:59:03,015 INFO [RS:0;037c58a19ddb:37267-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/f3ed23ac403772e0fe4ca9ee892544ec/B/121e6e78f34e432d92caf3c87f619196, hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/f3ed23ac403772e0fe4ca9ee892544ec/B/6b4c1d88775e40b89a0762d4905b7d9f, hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/f3ed23ac403772e0fe4ca9ee892544ec/B/edee38773ced4b9ea467a4d5655b6334, hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/f3ed23ac403772e0fe4ca9ee892544ec/B/7b8eb25c363c4c66a4eb1bc0ba18a151] into tmpdir=hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/f3ed23ac403772e0fe4ca9ee892544ec/.tmp, totalSize=46.9 K 2024-11-16T09:59:03,016 DEBUG [RS:0;037c58a19ddb:37267-shortCompactions-0 {}] compactions.Compactor(225): Compacting c97cbf9e65af4cc4afa063288dd53d11, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=13, earliestPutTs=1731751139749 2024-11-16T09:59:03,016 DEBUG [RS:0;037c58a19ddb:37267-longCompactions-0 {}] compactions.Compactor(225): Compacting 121e6e78f34e432d92caf3c87f619196, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=13, earliestPutTs=1731751139749 2024-11-16T09:59:03,016 DEBUG [RS:0;037c58a19ddb:37267-shortCompactions-0 {}] compactions.Compactor(225): Compacting c3e74913af05458d9452010d8e8044c0, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=37, earliestPutTs=1731751139767 2024-11-16T09:59:03,016 DEBUG [RS:0;037c58a19ddb:37267-longCompactions-0 {}] compactions.Compactor(225): Compacting 6b4c1d88775e40b89a0762d4905b7d9f, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=37, earliestPutTs=1731751139767 2024-11-16T09:59:03,017 DEBUG [RS:0;037c58a19ddb:37267-shortCompactions-0 {}] compactions.Compactor(225): Compacting 3dc937ecac70408cba3ef81b89802dd1, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=50, earliestPutTs=1731751140417 2024-11-16T09:59:03,017 DEBUG [RS:0;037c58a19ddb:37267-longCompactions-0 {}] compactions.Compactor(225): Compacting edee38773ced4b9ea467a4d5655b6334, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=50, earliestPutTs=1731751140417 2024-11-16T09:59:03,017 DEBUG [RS:0;037c58a19ddb:37267-shortCompactions-0 {}] compactions.Compactor(225): Compacting a4ceccf438df4ea487b2d78dc8baaad9, keycount=200, bloomtype=ROW, size=14.0 K, encoding=NONE, compression=NONE, seqNum=74, earliestPutTs=1731751141108 2024-11-16T09:59:03,017 DEBUG [RS:0;037c58a19ddb:37267-longCompactions-0 {}] compactions.Compactor(225): Compacting 7b8eb25c363c4c66a4eb1bc0ba18a151, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=74, earliestPutTs=1731751141108 2024-11-16T09:59:03,027 INFO [RS:0;037c58a19ddb:37267-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): f3ed23ac403772e0fe4ca9ee892544ec#B#compaction#222 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-16T09:59:03,027 DEBUG [RS:0;037c58a19ddb:37267-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/f3ed23ac403772e0fe4ca9ee892544ec/.tmp/B/1ab050ebce744bc993da534cbe3617aa is 50, key is test_row_0/B:col10/1731751141108/Put/seqid=0 2024-11-16T09:59:03,028 INFO [RS:0;037c58a19ddb:37267-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): f3ed23ac403772e0fe4ca9ee892544ec#A#compaction#223 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-16T09:59:03,029 DEBUG [RS:0;037c58a19ddb:37267-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/f3ed23ac403772e0fe4ca9ee892544ec/.tmp/A/c4ef90a8928e4ab49e0b5f591292b9b9 is 50, key is test_row_0/A:col10/1731751141108/Put/seqid=0 2024-11-16T09:59:03,033 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37591 is added to blk_1073742091_1267 (size=12139) 2024-11-16T09:59:03,037 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37591 is added to blk_1073742092_1268 (size=12139) 2024-11-16T09:59:03,097 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=37267 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=65 2024-11-16T09:59:03,098 DEBUG [RS_FLUSH_OPERATIONS-regionserver/037c58a19ddb:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731751137553.f3ed23ac403772e0fe4ca9ee892544ec. 2024-11-16T09:59:03,098 INFO [RS_FLUSH_OPERATIONS-regionserver/037c58a19ddb:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.HRegion(2902): Flushing f3ed23ac403772e0fe4ca9ee892544ec 3/3 column families, dataSize=60.38 KB heapSize=158.95 KB 2024-11-16T09:59:03,102 DEBUG [RS_FLUSH_OPERATIONS-regionserver/037c58a19ddb:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/f3ed23ac403772e0fe4ca9ee892544ec/.tmp/A/699d58ef21354836b7b3485f7bcfb2c0 is 50, key is test_row_0/A:col10/1731751141747/Put/seqid=0 2024-11-16T09:59:03,108 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37591 is added to blk_1073742093_1269 (size=12001) 2024-11-16T09:59:03,109 INFO [RS_FLUSH_OPERATIONS-regionserver/037c58a19ddb:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=86 (bloomFilter=true), to=hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/f3ed23ac403772e0fe4ca9ee892544ec/.tmp/A/699d58ef21354836b7b3485f7bcfb2c0 2024-11-16T09:59:03,117 DEBUG [RS_FLUSH_OPERATIONS-regionserver/037c58a19ddb:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/f3ed23ac403772e0fe4ca9ee892544ec/.tmp/B/2801d6506c4844e8ab66e1d8707f85ba is 50, key is test_row_0/B:col10/1731751141747/Put/seqid=0 2024-11-16T09:59:03,130 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37591 is added to blk_1073742094_1270 (size=12001) 2024-11-16T09:59:03,163 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37877 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=64 2024-11-16T09:59:03,441 DEBUG [RS:0;037c58a19ddb:37267-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/f3ed23ac403772e0fe4ca9ee892544ec/.tmp/B/1ab050ebce744bc993da534cbe3617aa as hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/f3ed23ac403772e0fe4ca9ee892544ec/B/1ab050ebce744bc993da534cbe3617aa 2024-11-16T09:59:03,448 DEBUG [RS:0;037c58a19ddb:37267-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/f3ed23ac403772e0fe4ca9ee892544ec/.tmp/A/c4ef90a8928e4ab49e0b5f591292b9b9 as hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/f3ed23ac403772e0fe4ca9ee892544ec/A/c4ef90a8928e4ab49e0b5f591292b9b9 2024-11-16T09:59:03,451 INFO [RS:0;037c58a19ddb:37267-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in f3ed23ac403772e0fe4ca9ee892544ec/B of f3ed23ac403772e0fe4ca9ee892544ec into 1ab050ebce744bc993da534cbe3617aa(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-16T09:59:03,451 DEBUG [RS:0;037c58a19ddb:37267-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for f3ed23ac403772e0fe4ca9ee892544ec: 2024-11-16T09:59:03,451 INFO [RS:0;037c58a19ddb:37267-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731751137553.f3ed23ac403772e0fe4ca9ee892544ec., storeName=f3ed23ac403772e0fe4ca9ee892544ec/B, priority=12, startTime=1731751143013; duration=0sec 2024-11-16T09:59:03,451 DEBUG [RS:0;037c58a19ddb:37267-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-16T09:59:03,451 DEBUG [RS:0;037c58a19ddb:37267-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: f3ed23ac403772e0fe4ca9ee892544ec:B 2024-11-16T09:59:03,452 DEBUG [RS:0;037c58a19ddb:37267-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-16T09:59:03,453 INFO [RS:0;037c58a19ddb:37267-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in f3ed23ac403772e0fe4ca9ee892544ec/A of f3ed23ac403772e0fe4ca9ee892544ec into c4ef90a8928e4ab49e0b5f591292b9b9(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-16T09:59:03,453 DEBUG [RS:0;037c58a19ddb:37267-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for f3ed23ac403772e0fe4ca9ee892544ec: 2024-11-16T09:59:03,453 INFO [RS:0;037c58a19ddb:37267-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731751137553.f3ed23ac403772e0fe4ca9ee892544ec., storeName=f3ed23ac403772e0fe4ca9ee892544ec/A, priority=12, startTime=1731751143013; duration=0sec 2024-11-16T09:59:03,453 DEBUG [RS:0;037c58a19ddb:37267-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-16T09:59:03,454 DEBUG [RS:0;037c58a19ddb:37267-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: f3ed23ac403772e0fe4ca9ee892544ec:A 2024-11-16T09:59:03,454 DEBUG [RS:0;037c58a19ddb:37267-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48004 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-16T09:59:03,454 DEBUG [RS:0;037c58a19ddb:37267-longCompactions-0 {}] regionserver.HStore(1541): f3ed23ac403772e0fe4ca9ee892544ec/C is initiating minor compaction (all files) 2024-11-16T09:59:03,454 INFO [RS:0;037c58a19ddb:37267-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of f3ed23ac403772e0fe4ca9ee892544ec/C in TestAcidGuarantees,,1731751137553.f3ed23ac403772e0fe4ca9ee892544ec. 2024-11-16T09:59:03,454 INFO [RS:0;037c58a19ddb:37267-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/f3ed23ac403772e0fe4ca9ee892544ec/C/f5e10ca7ce7f4643ac9f2b880087ce59, hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/f3ed23ac403772e0fe4ca9ee892544ec/C/03ddda3dad844fc09f3e33be2515e85e, hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/f3ed23ac403772e0fe4ca9ee892544ec/C/b747dafa9b994f2ba6ec3cb9ada2e63d, hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/f3ed23ac403772e0fe4ca9ee892544ec/C/b20aebb0d817468f8d6fb345deac0dc9] into tmpdir=hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/f3ed23ac403772e0fe4ca9ee892544ec/.tmp, totalSize=46.9 K 2024-11-16T09:59:03,455 DEBUG [RS:0;037c58a19ddb:37267-longCompactions-0 {}] compactions.Compactor(225): Compacting f5e10ca7ce7f4643ac9f2b880087ce59, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=13, earliestPutTs=1731751139749 2024-11-16T09:59:03,455 DEBUG [RS:0;037c58a19ddb:37267-longCompactions-0 {}] compactions.Compactor(225): Compacting 03ddda3dad844fc09f3e33be2515e85e, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=37, earliestPutTs=1731751139767 2024-11-16T09:59:03,455 DEBUG [RS:0;037c58a19ddb:37267-longCompactions-0 {}] compactions.Compactor(225): Compacting b747dafa9b994f2ba6ec3cb9ada2e63d, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=50, earliestPutTs=1731751140417 2024-11-16T09:59:03,456 DEBUG [RS:0;037c58a19ddb:37267-longCompactions-0 {}] compactions.Compactor(225): Compacting b20aebb0d817468f8d6fb345deac0dc9, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=74, earliestPutTs=1731751141108 2024-11-16T09:59:03,466 INFO [RS:0;037c58a19ddb:37267-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): f3ed23ac403772e0fe4ca9ee892544ec#C#compaction#226 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-16T09:59:03,467 DEBUG [RS:0;037c58a19ddb:37267-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/f3ed23ac403772e0fe4ca9ee892544ec/.tmp/C/4336fc254c134e0b93fa1380900fa45f is 50, key is test_row_0/C:col10/1731751141108/Put/seqid=0 2024-11-16T09:59:03,475 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37591 is added to blk_1073742095_1271 (size=12139) 2024-11-16T09:59:03,482 DEBUG [RS:0;037c58a19ddb:37267-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/f3ed23ac403772e0fe4ca9ee892544ec/.tmp/C/4336fc254c134e0b93fa1380900fa45f as hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/f3ed23ac403772e0fe4ca9ee892544ec/C/4336fc254c134e0b93fa1380900fa45f 2024-11-16T09:59:03,488 INFO [RS:0;037c58a19ddb:37267-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in f3ed23ac403772e0fe4ca9ee892544ec/C of f3ed23ac403772e0fe4ca9ee892544ec into 4336fc254c134e0b93fa1380900fa45f(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-16T09:59:03,488 DEBUG [RS:0;037c58a19ddb:37267-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for f3ed23ac403772e0fe4ca9ee892544ec: 2024-11-16T09:59:03,488 INFO [RS:0;037c58a19ddb:37267-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731751137553.f3ed23ac403772e0fe4ca9ee892544ec., storeName=f3ed23ac403772e0fe4ca9ee892544ec/C, priority=12, startTime=1731751143014; duration=0sec 2024-11-16T09:59:03,488 DEBUG [RS:0;037c58a19ddb:37267-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-16T09:59:03,488 DEBUG [RS:0;037c58a19ddb:37267-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: f3ed23ac403772e0fe4ca9ee892544ec:C 2024-11-16T09:59:03,531 INFO [RS_FLUSH_OPERATIONS-regionserver/037c58a19ddb:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=86 (bloomFilter=true), to=hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/f3ed23ac403772e0fe4ca9ee892544ec/.tmp/B/2801d6506c4844e8ab66e1d8707f85ba 2024-11-16T09:59:03,539 DEBUG [RS_FLUSH_OPERATIONS-regionserver/037c58a19ddb:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/f3ed23ac403772e0fe4ca9ee892544ec/.tmp/C/7a98e6e6c80d4413a5a5f621e7789567 is 50, key is test_row_0/C:col10/1731751141747/Put/seqid=0 2024-11-16T09:59:03,549 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37591 is added to blk_1073742096_1272 (size=12001) 2024-11-16T09:59:03,924 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] regionserver.HRegion(8855): Flush requested on f3ed23ac403772e0fe4ca9ee892544ec 2024-11-16T09:59:03,924 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731751137553.f3ed23ac403772e0fe4ca9ee892544ec. as already flushing 2024-11-16T09:59:03,951 INFO [RS_FLUSH_OPERATIONS-regionserver/037c58a19ddb:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=86 (bloomFilter=true), to=hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/f3ed23ac403772e0fe4ca9ee892544ec/.tmp/C/7a98e6e6c80d4413a5a5f621e7789567 2024-11-16T09:59:03,956 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3ed23ac403772e0fe4ca9ee892544ec, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T09:59:03,956 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34386 deadline: 1731751203951, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3ed23ac403772e0fe4ca9ee892544ec, server=037c58a19ddb,37267,1731751077844 2024-11-16T09:59:03,956 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731751137553.f3ed23ac403772e0fe4ca9ee892544ec., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 , the old value is region=TestAcidGuarantees,,1731751137553.f3ed23ac403772e0fe4ca9ee892544ec., hostname=037c58a19ddb,37267,1731751077844, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3ed23ac403772e0fe4ca9ee892544ec, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:59:03,956 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731751137553.f3ed23ac403772e0fe4ca9ee892544ec., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3ed23ac403772e0fe4ca9ee892544ec, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:59:03,956 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731751137553.f3ed23ac403772e0fe4ca9ee892544ec., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 because the exception is null or not the one we care about 2024-11-16T09:59:03,957 DEBUG [RS_FLUSH_OPERATIONS-regionserver/037c58a19ddb:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/f3ed23ac403772e0fe4ca9ee892544ec/.tmp/A/699d58ef21354836b7b3485f7bcfb2c0 as hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/f3ed23ac403772e0fe4ca9ee892544ec/A/699d58ef21354836b7b3485f7bcfb2c0 2024-11-16T09:59:03,958 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3ed23ac403772e0fe4ca9ee892544ec, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T09:59:03,958 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34414 deadline: 1731751203953, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3ed23ac403772e0fe4ca9ee892544ec, server=037c58a19ddb,37267,1731751077844 2024-11-16T09:59:03,958 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3ed23ac403772e0fe4ca9ee892544ec, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T09:59:03,958 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731751137553.f3ed23ac403772e0fe4ca9ee892544ec., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 , the old value is region=TestAcidGuarantees,,1731751137553.f3ed23ac403772e0fe4ca9ee892544ec., hostname=037c58a19ddb,37267,1731751077844, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3ed23ac403772e0fe4ca9ee892544ec, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:59:03,959 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731751137553.f3ed23ac403772e0fe4ca9ee892544ec., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3ed23ac403772e0fe4ca9ee892544ec, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:59:03,959 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34368 deadline: 1731751203954, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3ed23ac403772e0fe4ca9ee892544ec, server=037c58a19ddb,37267,1731751077844 2024-11-16T09:59:03,959 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731751137553.f3ed23ac403772e0fe4ca9ee892544ec., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 because the exception is null or not the one we care about 2024-11-16T09:59:03,959 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3ed23ac403772e0fe4ca9ee892544ec, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T09:59:03,959 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731751137553.f3ed23ac403772e0fe4ca9ee892544ec., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 , the old value is region=TestAcidGuarantees,,1731751137553.f3ed23ac403772e0fe4ca9ee892544ec., hostname=037c58a19ddb,37267,1731751077844, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3ed23ac403772e0fe4ca9ee892544ec, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:59:03,959 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731751137553.f3ed23ac403772e0fe4ca9ee892544ec., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3ed23ac403772e0fe4ca9ee892544ec, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:59:03,959 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731751137553.f3ed23ac403772e0fe4ca9ee892544ec., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 because the exception is null or not the one we care about 2024-11-16T09:59:03,959 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34408 deadline: 1731751203955, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3ed23ac403772e0fe4ca9ee892544ec, server=037c58a19ddb,37267,1731751077844 2024-11-16T09:59:03,959 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731751137553.f3ed23ac403772e0fe4ca9ee892544ec., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 , the old value is region=TestAcidGuarantees,,1731751137553.f3ed23ac403772e0fe4ca9ee892544ec., hostname=037c58a19ddb,37267,1731751077844, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3ed23ac403772e0fe4ca9ee892544ec, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:59:03,959 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731751137553.f3ed23ac403772e0fe4ca9ee892544ec., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3ed23ac403772e0fe4ca9ee892544ec, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:59:03,959 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731751137553.f3ed23ac403772e0fe4ca9ee892544ec., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 because the exception is null or not the one we care about 2024-11-16T09:59:03,960 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3ed23ac403772e0fe4ca9ee892544ec, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T09:59:03,960 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34382 deadline: 1731751203960, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3ed23ac403772e0fe4ca9ee892544ec, server=037c58a19ddb,37267,1731751077844 2024-11-16T09:59:03,961 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731751137553.f3ed23ac403772e0fe4ca9ee892544ec., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 , the old value is region=TestAcidGuarantees,,1731751137553.f3ed23ac403772e0fe4ca9ee892544ec., hostname=037c58a19ddb,37267,1731751077844, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3ed23ac403772e0fe4ca9ee892544ec, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:59:03,961 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731751137553.f3ed23ac403772e0fe4ca9ee892544ec., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3ed23ac403772e0fe4ca9ee892544ec, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:59:03,962 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731751137553.f3ed23ac403772e0fe4ca9ee892544ec., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 because the exception is null or not the one we care about 2024-11-16T09:59:03,962 INFO [RS_FLUSH_OPERATIONS-regionserver/037c58a19ddb:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/f3ed23ac403772e0fe4ca9ee892544ec/A/699d58ef21354836b7b3485f7bcfb2c0, entries=150, sequenceid=86, filesize=11.7 K 2024-11-16T09:59:03,963 DEBUG [RS_FLUSH_OPERATIONS-regionserver/037c58a19ddb:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/f3ed23ac403772e0fe4ca9ee892544ec/.tmp/B/2801d6506c4844e8ab66e1d8707f85ba as hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/f3ed23ac403772e0fe4ca9ee892544ec/B/2801d6506c4844e8ab66e1d8707f85ba 2024-11-16T09:59:03,967 INFO [RS_FLUSH_OPERATIONS-regionserver/037c58a19ddb:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/f3ed23ac403772e0fe4ca9ee892544ec/B/2801d6506c4844e8ab66e1d8707f85ba, entries=150, sequenceid=86, filesize=11.7 K 2024-11-16T09:59:03,968 DEBUG [RS_FLUSH_OPERATIONS-regionserver/037c58a19ddb:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/f3ed23ac403772e0fe4ca9ee892544ec/.tmp/C/7a98e6e6c80d4413a5a5f621e7789567 as hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/f3ed23ac403772e0fe4ca9ee892544ec/C/7a98e6e6c80d4413a5a5f621e7789567 2024-11-16T09:59:03,972 INFO [RS_FLUSH_OPERATIONS-regionserver/037c58a19ddb:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/f3ed23ac403772e0fe4ca9ee892544ec/C/7a98e6e6c80d4413a5a5f621e7789567, entries=150, sequenceid=86, filesize=11.7 K 2024-11-16T09:59:03,973 INFO [RS_FLUSH_OPERATIONS-regionserver/037c58a19ddb:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~158.91 KB/162720, currentSize=147.60 KB/151140 for f3ed23ac403772e0fe4ca9ee892544ec in 874ms, sequenceid=86, compaction requested=false 2024-11-16T09:59:03,973 DEBUG [RS_FLUSH_OPERATIONS-regionserver/037c58a19ddb:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.HRegion(2603): Flush status journal for f3ed23ac403772e0fe4ca9ee892544ec: 2024-11-16T09:59:03,973 DEBUG [RS_FLUSH_OPERATIONS-regionserver/037c58a19ddb:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731751137553.f3ed23ac403772e0fe4ca9ee892544ec. 2024-11-16T09:59:03,973 DEBUG [RS_FLUSH_OPERATIONS-regionserver/037c58a19ddb:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=65 2024-11-16T09:59:03,973 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37877 {}] master.HMaster(4169): Remote procedure done, pid=65 2024-11-16T09:59:03,976 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=65, resume processing ppid=64 2024-11-16T09:59:03,977 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=65, ppid=64, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.9480 sec 2024-11-16T09:59:03,979 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=64, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=64, table=TestAcidGuarantees in 1.9540 sec 2024-11-16T09:59:04,065 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] regionserver.HRegion(8855): Flush requested on f3ed23ac403772e0fe4ca9ee892544ec 2024-11-16T09:59:04,066 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-16T09:59:04,066 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing f3ed23ac403772e0fe4ca9ee892544ec 3/3 column families, dataSize=154.31 KB heapSize=405.05 KB 2024-11-16T09:59:04,073 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/f3ed23ac403772e0fe4ca9ee892544ec/.tmp/A/27f0ac8785f94a86aedae7fb6146d397 is 50, key is test_row_0/A:col10/1731751143951/Put/seqid=0 2024-11-16T09:59:04,077 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37591 is added to blk_1073742097_1273 (size=12001) 2024-11-16T09:59:04,078 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3ed23ac403772e0fe4ca9ee892544ec, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T09:59:04,078 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34386 deadline: 1731751204073, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3ed23ac403772e0fe4ca9ee892544ec, server=037c58a19ddb,37267,1731751077844 2024-11-16T09:59:04,079 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731751137553.f3ed23ac403772e0fe4ca9ee892544ec., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 , the old value is region=TestAcidGuarantees,,1731751137553.f3ed23ac403772e0fe4ca9ee892544ec., hostname=037c58a19ddb,37267,1731751077844, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3ed23ac403772e0fe4ca9ee892544ec, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:59:04,079 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731751137553.f3ed23ac403772e0fe4ca9ee892544ec., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3ed23ac403772e0fe4ca9ee892544ec, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:59:04,079 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731751137553.f3ed23ac403772e0fe4ca9ee892544ec., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 because the exception is null or not the one we care about 2024-11-16T09:59:04,080 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3ed23ac403772e0fe4ca9ee892544ec, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T09:59:04,080 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34414 deadline: 1731751204075, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3ed23ac403772e0fe4ca9ee892544ec, server=037c58a19ddb,37267,1731751077844 2024-11-16T09:59:04,080 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731751137553.f3ed23ac403772e0fe4ca9ee892544ec., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 , the old value is region=TestAcidGuarantees,,1731751137553.f3ed23ac403772e0fe4ca9ee892544ec., hostname=037c58a19ddb,37267,1731751077844, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3ed23ac403772e0fe4ca9ee892544ec, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:59:04,080 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3ed23ac403772e0fe4ca9ee892544ec, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T09:59:04,080 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731751137553.f3ed23ac403772e0fe4ca9ee892544ec., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3ed23ac403772e0fe4ca9ee892544ec, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:59:04,080 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731751137553.f3ed23ac403772e0fe4ca9ee892544ec., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 because the exception is null or not the one we care about 2024-11-16T09:59:04,080 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34408 deadline: 1731751204075, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3ed23ac403772e0fe4ca9ee892544ec, server=037c58a19ddb,37267,1731751077844 2024-11-16T09:59:04,081 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731751137553.f3ed23ac403772e0fe4ca9ee892544ec., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 , the old value is region=TestAcidGuarantees,,1731751137553.f3ed23ac403772e0fe4ca9ee892544ec., hostname=037c58a19ddb,37267,1731751077844, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3ed23ac403772e0fe4ca9ee892544ec, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:59:04,081 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3ed23ac403772e0fe4ca9ee892544ec, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T09:59:04,081 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731751137553.f3ed23ac403772e0fe4ca9ee892544ec., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3ed23ac403772e0fe4ca9ee892544ec, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:59:04,081 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731751137553.f3ed23ac403772e0fe4ca9ee892544ec., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 because the exception is null or not the one we care about 2024-11-16T09:59:04,081 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34382 deadline: 1731751204076, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3ed23ac403772e0fe4ca9ee892544ec, server=037c58a19ddb,37267,1731751077844 2024-11-16T09:59:04,081 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731751137553.f3ed23ac403772e0fe4ca9ee892544ec., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 , the old value is region=TestAcidGuarantees,,1731751137553.f3ed23ac403772e0fe4ca9ee892544ec., hostname=037c58a19ddb,37267,1731751077844, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3ed23ac403772e0fe4ca9ee892544ec, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:59:04,081 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3ed23ac403772e0fe4ca9ee892544ec, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T09:59:04,081 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731751137553.f3ed23ac403772e0fe4ca9ee892544ec., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3ed23ac403772e0fe4ca9ee892544ec, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:59:04,081 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731751137553.f3ed23ac403772e0fe4ca9ee892544ec., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 because the exception is null or not the one we care about 2024-11-16T09:59:04,081 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34368 deadline: 1731751204076, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3ed23ac403772e0fe4ca9ee892544ec, server=037c58a19ddb,37267,1731751077844 2024-11-16T09:59:04,081 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731751137553.f3ed23ac403772e0fe4ca9ee892544ec., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 , the old value is region=TestAcidGuarantees,,1731751137553.f3ed23ac403772e0fe4ca9ee892544ec., hostname=037c58a19ddb,37267,1731751077844, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3ed23ac403772e0fe4ca9ee892544ec, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:59:04,081 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731751137553.f3ed23ac403772e0fe4ca9ee892544ec., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3ed23ac403772e0fe4ca9ee892544ec, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:59:04,081 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731751137553.f3ed23ac403772e0fe4ca9ee892544ec., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 because the exception is null or not the one we care about 2024-11-16T09:59:04,173 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37877 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=64 2024-11-16T09:59:04,173 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-16T09:59:04,173 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37877 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-16T09:59:04,175 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37877 {}] procedure2.ProcedureExecutor(1139): Stored pid=66, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=66, table=TestAcidGuarantees 2024-11-16T09:59:04,175 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37877 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=66 2024-11-16T09:59:04,176 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=66, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=66, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-16T09:59:04,177 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=66, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=66, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-16T09:59:04,177 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=67, ppid=66, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-16T09:59:04,184 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3ed23ac403772e0fe4ca9ee892544ec, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T09:59:04,185 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34386 deadline: 1731751204183, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3ed23ac403772e0fe4ca9ee892544ec, server=037c58a19ddb,37267,1731751077844 2024-11-16T09:59:04,185 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3ed23ac403772e0fe4ca9ee892544ec, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T09:59:04,185 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731751137553.f3ed23ac403772e0fe4ca9ee892544ec., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 , the old value is region=TestAcidGuarantees,,1731751137553.f3ed23ac403772e0fe4ca9ee892544ec., hostname=037c58a19ddb,37267,1731751077844, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3ed23ac403772e0fe4ca9ee892544ec, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:59:04,185 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731751137553.f3ed23ac403772e0fe4ca9ee892544ec., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3ed23ac403772e0fe4ca9ee892544ec, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:59:04,185 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731751137553.f3ed23ac403772e0fe4ca9ee892544ec., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 because the exception is null or not the one we care about 2024-11-16T09:59:04,185 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34408 deadline: 1731751204183, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3ed23ac403772e0fe4ca9ee892544ec, server=037c58a19ddb,37267,1731751077844 2024-11-16T09:59:04,185 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3ed23ac403772e0fe4ca9ee892544ec, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T09:59:04,185 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731751137553.f3ed23ac403772e0fe4ca9ee892544ec., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 , the old value is region=TestAcidGuarantees,,1731751137553.f3ed23ac403772e0fe4ca9ee892544ec., hostname=037c58a19ddb,37267,1731751077844, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3ed23ac403772e0fe4ca9ee892544ec, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:59:04,185 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731751137553.f3ed23ac403772e0fe4ca9ee892544ec., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3ed23ac403772e0fe4ca9ee892544ec, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:59:04,185 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34414 deadline: 1731751204183, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3ed23ac403772e0fe4ca9ee892544ec, server=037c58a19ddb,37267,1731751077844 2024-11-16T09:59:04,185 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731751137553.f3ed23ac403772e0fe4ca9ee892544ec., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 because the exception is null or not the one we care about 2024-11-16T09:59:04,186 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3ed23ac403772e0fe4ca9ee892544ec, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T09:59:04,186 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731751137553.f3ed23ac403772e0fe4ca9ee892544ec., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 , the old value is region=TestAcidGuarantees,,1731751137553.f3ed23ac403772e0fe4ca9ee892544ec., hostname=037c58a19ddb,37267,1731751077844, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3ed23ac403772e0fe4ca9ee892544ec, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:59:04,186 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731751137553.f3ed23ac403772e0fe4ca9ee892544ec., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3ed23ac403772e0fe4ca9ee892544ec, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:59:04,186 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34382 deadline: 1731751204183, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3ed23ac403772e0fe4ca9ee892544ec, server=037c58a19ddb,37267,1731751077844 2024-11-16T09:59:04,186 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731751137553.f3ed23ac403772e0fe4ca9ee892544ec., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 because the exception is null or not the one we care about 2024-11-16T09:59:04,186 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731751137553.f3ed23ac403772e0fe4ca9ee892544ec., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 , the old value is region=TestAcidGuarantees,,1731751137553.f3ed23ac403772e0fe4ca9ee892544ec., hostname=037c58a19ddb,37267,1731751077844, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3ed23ac403772e0fe4ca9ee892544ec, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:59:04,186 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731751137553.f3ed23ac403772e0fe4ca9ee892544ec., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3ed23ac403772e0fe4ca9ee892544ec, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:59:04,186 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731751137553.f3ed23ac403772e0fe4ca9ee892544ec., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 because the exception is null or not the one we care about 2024-11-16T09:59:04,194 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3ed23ac403772e0fe4ca9ee892544ec, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T09:59:04,195 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34368 deadline: 1731751204192, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3ed23ac403772e0fe4ca9ee892544ec, server=037c58a19ddb,37267,1731751077844 2024-11-16T09:59:04,195 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731751137553.f3ed23ac403772e0fe4ca9ee892544ec., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 , the old value is region=TestAcidGuarantees,,1731751137553.f3ed23ac403772e0fe4ca9ee892544ec., hostname=037c58a19ddb,37267,1731751077844, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3ed23ac403772e0fe4ca9ee892544ec, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:59:04,195 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731751137553.f3ed23ac403772e0fe4ca9ee892544ec., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3ed23ac403772e0fe4ca9ee892544ec, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:59:04,195 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731751137553.f3ed23ac403772e0fe4ca9ee892544ec., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 because the exception is null or not the one we care about 2024-11-16T09:59:04,283 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37877 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=66 2024-11-16T09:59:04,330 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=37267 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=67 2024-11-16T09:59:04,330 DEBUG [RS_FLUSH_OPERATIONS-regionserver/037c58a19ddb:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731751137553.f3ed23ac403772e0fe4ca9ee892544ec. 2024-11-16T09:59:04,330 DEBUG [RS_FLUSH_OPERATIONS-regionserver/037c58a19ddb:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731751137553.f3ed23ac403772e0fe4ca9ee892544ec. as already flushing 2024-11-16T09:59:04,330 DEBUG [RS_FLUSH_OPERATIONS-regionserver/037c58a19ddb:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731751137553.f3ed23ac403772e0fe4ca9ee892544ec. 2024-11-16T09:59:04,330 ERROR [RS_FLUSH_OPERATIONS-regionserver/037c58a19ddb:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] handler.RSProcedureHandler(58): pid=67 java.io.IOException: Unable to complete flush {ENCODED => f3ed23ac403772e0fe4ca9ee892544ec, NAME => 'TestAcidGuarantees,,1731751137553.f3ed23ac403772e0fe4ca9ee892544ec.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T09:59:04,330 DEBUG [RS_FLUSH_OPERATIONS-regionserver/037c58a19ddb:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=67 java.io.IOException: Unable to complete flush {ENCODED => f3ed23ac403772e0fe4ca9ee892544ec, NAME => 'TestAcidGuarantees,,1731751137553.f3ed23ac403772e0fe4ca9ee892544ec.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T09:59:04,331 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37877 {}] master.HMaster(4177): Remote procedure failed, pid=67 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => f3ed23ac403772e0fe4ca9ee892544ec, NAME => 'TestAcidGuarantees,,1731751137553.f3ed23ac403772e0fe4ca9ee892544ec.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => f3ed23ac403772e0fe4ca9ee892544ec, NAME => 'TestAcidGuarantees,,1731751137553.f3ed23ac403772e0fe4ca9ee892544ec.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T09:59:04,395 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3ed23ac403772e0fe4ca9ee892544ec, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T09:59:04,396 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34386 deadline: 1731751204393, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3ed23ac403772e0fe4ca9ee892544ec, server=037c58a19ddb,37267,1731751077844 2024-11-16T09:59:04,396 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731751137553.f3ed23ac403772e0fe4ca9ee892544ec., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 , the old value is region=TestAcidGuarantees,,1731751137553.f3ed23ac403772e0fe4ca9ee892544ec., hostname=037c58a19ddb,37267,1731751077844, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3ed23ac403772e0fe4ca9ee892544ec, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:59:04,396 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731751137553.f3ed23ac403772e0fe4ca9ee892544ec., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3ed23ac403772e0fe4ca9ee892544ec, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:59:04,396 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731751137553.f3ed23ac403772e0fe4ca9ee892544ec., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 because the exception is null or not the one we care about 2024-11-16T09:59:04,396 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3ed23ac403772e0fe4ca9ee892544ec, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T09:59:04,396 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34408 deadline: 1731751204393, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3ed23ac403772e0fe4ca9ee892544ec, server=037c58a19ddb,37267,1731751077844 2024-11-16T09:59:04,396 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3ed23ac403772e0fe4ca9ee892544ec, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T09:59:04,397 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731751137553.f3ed23ac403772e0fe4ca9ee892544ec., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 , the old value is region=TestAcidGuarantees,,1731751137553.f3ed23ac403772e0fe4ca9ee892544ec., hostname=037c58a19ddb,37267,1731751077844, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3ed23ac403772e0fe4ca9ee892544ec, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:59:04,397 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34414 deadline: 1731751204393, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3ed23ac403772e0fe4ca9ee892544ec, server=037c58a19ddb,37267,1731751077844 2024-11-16T09:59:04,397 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731751137553.f3ed23ac403772e0fe4ca9ee892544ec., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3ed23ac403772e0fe4ca9ee892544ec, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:59:04,397 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731751137553.f3ed23ac403772e0fe4ca9ee892544ec., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 because the exception is null or not the one we care about 2024-11-16T09:59:04,397 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3ed23ac403772e0fe4ca9ee892544ec, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T09:59:04,397 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731751137553.f3ed23ac403772e0fe4ca9ee892544ec., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 , the old value is region=TestAcidGuarantees,,1731751137553.f3ed23ac403772e0fe4ca9ee892544ec., hostname=037c58a19ddb,37267,1731751077844, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3ed23ac403772e0fe4ca9ee892544ec, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:59:04,397 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731751137553.f3ed23ac403772e0fe4ca9ee892544ec., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3ed23ac403772e0fe4ca9ee892544ec, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:59:04,397 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34382 deadline: 1731751204393, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3ed23ac403772e0fe4ca9ee892544ec, server=037c58a19ddb,37267,1731751077844 2024-11-16T09:59:04,397 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731751137553.f3ed23ac403772e0fe4ca9ee892544ec., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 because the exception is null or not the one we care about 2024-11-16T09:59:04,397 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731751137553.f3ed23ac403772e0fe4ca9ee892544ec., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 , the old value is region=TestAcidGuarantees,,1731751137553.f3ed23ac403772e0fe4ca9ee892544ec., hostname=037c58a19ddb,37267,1731751077844, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3ed23ac403772e0fe4ca9ee892544ec, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:59:04,397 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731751137553.f3ed23ac403772e0fe4ca9ee892544ec., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3ed23ac403772e0fe4ca9ee892544ec, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:59:04,397 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731751137553.f3ed23ac403772e0fe4ca9ee892544ec., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 because the exception is null or not the one we care about 2024-11-16T09:59:04,406 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3ed23ac403772e0fe4ca9ee892544ec, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T09:59:04,407 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34368 deadline: 1731751204403, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3ed23ac403772e0fe4ca9ee892544ec, server=037c58a19ddb,37267,1731751077844 2024-11-16T09:59:04,407 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731751137553.f3ed23ac403772e0fe4ca9ee892544ec., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 , the old value is region=TestAcidGuarantees,,1731751137553.f3ed23ac403772e0fe4ca9ee892544ec., hostname=037c58a19ddb,37267,1731751077844, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3ed23ac403772e0fe4ca9ee892544ec, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:59:04,407 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731751137553.f3ed23ac403772e0fe4ca9ee892544ec., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3ed23ac403772e0fe4ca9ee892544ec, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:59:04,407 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731751137553.f3ed23ac403772e0fe4ca9ee892544ec., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 because the exception is null or not the one we care about 2024-11-16T09:59:04,478 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=53.67 KB at sequenceid=116 (bloomFilter=true), to=hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/f3ed23ac403772e0fe4ca9ee892544ec/.tmp/A/27f0ac8785f94a86aedae7fb6146d397 2024-11-16T09:59:04,483 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=37267 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=67 2024-11-16T09:59:04,483 DEBUG [RS_FLUSH_OPERATIONS-regionserver/037c58a19ddb:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731751137553.f3ed23ac403772e0fe4ca9ee892544ec. 2024-11-16T09:59:04,483 DEBUG [RS_FLUSH_OPERATIONS-regionserver/037c58a19ddb:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731751137553.f3ed23ac403772e0fe4ca9ee892544ec. as already flushing 2024-11-16T09:59:04,483 DEBUG [RS_FLUSH_OPERATIONS-regionserver/037c58a19ddb:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731751137553.f3ed23ac403772e0fe4ca9ee892544ec. 2024-11-16T09:59:04,483 ERROR [RS_FLUSH_OPERATIONS-regionserver/037c58a19ddb:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] handler.RSProcedureHandler(58): pid=67 java.io.IOException: Unable to complete flush {ENCODED => f3ed23ac403772e0fe4ca9ee892544ec, NAME => 'TestAcidGuarantees,,1731751137553.f3ed23ac403772e0fe4ca9ee892544ec.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T09:59:04,483 DEBUG [RS_FLUSH_OPERATIONS-regionserver/037c58a19ddb:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=67 java.io.IOException: Unable to complete flush {ENCODED => f3ed23ac403772e0fe4ca9ee892544ec, NAME => 'TestAcidGuarantees,,1731751137553.f3ed23ac403772e0fe4ca9ee892544ec.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T09:59:04,484 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37877 {}] master.HMaster(4177): Remote procedure failed, pid=67 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => f3ed23ac403772e0fe4ca9ee892544ec, NAME => 'TestAcidGuarantees,,1731751137553.f3ed23ac403772e0fe4ca9ee892544ec.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => f3ed23ac403772e0fe4ca9ee892544ec, NAME => 'TestAcidGuarantees,,1731751137553.f3ed23ac403772e0fe4ca9ee892544ec.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T09:59:04,487 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/f3ed23ac403772e0fe4ca9ee892544ec/.tmp/B/0500168b3f534c849c2134d112232df9 is 50, key is test_row_0/B:col10/1731751143951/Put/seqid=0 2024-11-16T09:59:04,492 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37591 is added to blk_1073742098_1274 (size=12001) 2024-11-16T09:59:04,493 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37877 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=66 2024-11-16T09:59:04,639 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=37267 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=67 2024-11-16T09:59:04,639 DEBUG [RS_FLUSH_OPERATIONS-regionserver/037c58a19ddb:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731751137553.f3ed23ac403772e0fe4ca9ee892544ec. 2024-11-16T09:59:04,639 DEBUG [RS_FLUSH_OPERATIONS-regionserver/037c58a19ddb:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731751137553.f3ed23ac403772e0fe4ca9ee892544ec. as already flushing 2024-11-16T09:59:04,639 DEBUG [RS_FLUSH_OPERATIONS-regionserver/037c58a19ddb:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731751137553.f3ed23ac403772e0fe4ca9ee892544ec. 2024-11-16T09:59:04,639 ERROR [RS_FLUSH_OPERATIONS-regionserver/037c58a19ddb:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] handler.RSProcedureHandler(58): pid=67 java.io.IOException: Unable to complete flush {ENCODED => f3ed23ac403772e0fe4ca9ee892544ec, NAME => 'TestAcidGuarantees,,1731751137553.f3ed23ac403772e0fe4ca9ee892544ec.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T09:59:04,639 DEBUG [RS_FLUSH_OPERATIONS-regionserver/037c58a19ddb:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=67 java.io.IOException: Unable to complete flush {ENCODED => f3ed23ac403772e0fe4ca9ee892544ec, NAME => 'TestAcidGuarantees,,1731751137553.f3ed23ac403772e0fe4ca9ee892544ec.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T09:59:04,640 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37877 {}] master.HMaster(4177): Remote procedure failed, pid=67 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => f3ed23ac403772e0fe4ca9ee892544ec, NAME => 'TestAcidGuarantees,,1731751137553.f3ed23ac403772e0fe4ca9ee892544ec.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => f3ed23ac403772e0fe4ca9ee892544ec, NAME => 'TestAcidGuarantees,,1731751137553.f3ed23ac403772e0fe4ca9ee892544ec.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T09:59:04,706 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3ed23ac403772e0fe4ca9ee892544ec, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T09:59:04,706 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34386 deadline: 1731751204703, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3ed23ac403772e0fe4ca9ee892544ec, server=037c58a19ddb,37267,1731751077844 2024-11-16T09:59:04,706 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3ed23ac403772e0fe4ca9ee892544ec, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T09:59:04,706 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731751137553.f3ed23ac403772e0fe4ca9ee892544ec., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 , the old value is region=TestAcidGuarantees,,1731751137553.f3ed23ac403772e0fe4ca9ee892544ec., hostname=037c58a19ddb,37267,1731751077844, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3ed23ac403772e0fe4ca9ee892544ec, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:59:04,706 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34408 deadline: 1731751204703, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3ed23ac403772e0fe4ca9ee892544ec, server=037c58a19ddb,37267,1731751077844 2024-11-16T09:59:04,706 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731751137553.f3ed23ac403772e0fe4ca9ee892544ec., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3ed23ac403772e0fe4ca9ee892544ec, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:59:04,706 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731751137553.f3ed23ac403772e0fe4ca9ee892544ec., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 because the exception is null or not the one we care about 2024-11-16T09:59:04,707 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3ed23ac403772e0fe4ca9ee892544ec, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T09:59:04,707 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731751137553.f3ed23ac403772e0fe4ca9ee892544ec., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 , the old value is region=TestAcidGuarantees,,1731751137553.f3ed23ac403772e0fe4ca9ee892544ec., hostname=037c58a19ddb,37267,1731751077844, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3ed23ac403772e0fe4ca9ee892544ec, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:59:04,707 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731751137553.f3ed23ac403772e0fe4ca9ee892544ec., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3ed23ac403772e0fe4ca9ee892544ec, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:59:04,707 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34414 deadline: 1731751204703, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3ed23ac403772e0fe4ca9ee892544ec, server=037c58a19ddb,37267,1731751077844 2024-11-16T09:59:04,707 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731751137553.f3ed23ac403772e0fe4ca9ee892544ec., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 because the exception is null or not the one we care about 2024-11-16T09:59:04,707 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3ed23ac403772e0fe4ca9ee892544ec, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T09:59:04,707 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731751137553.f3ed23ac403772e0fe4ca9ee892544ec., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 , the old value is region=TestAcidGuarantees,,1731751137553.f3ed23ac403772e0fe4ca9ee892544ec., hostname=037c58a19ddb,37267,1731751077844, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3ed23ac403772e0fe4ca9ee892544ec, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:59:04,707 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731751137553.f3ed23ac403772e0fe4ca9ee892544ec., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3ed23ac403772e0fe4ca9ee892544ec, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:59:04,707 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34382 deadline: 1731751204704, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3ed23ac403772e0fe4ca9ee892544ec, server=037c58a19ddb,37267,1731751077844 2024-11-16T09:59:04,707 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731751137553.f3ed23ac403772e0fe4ca9ee892544ec., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 because the exception is null or not the one we care about 2024-11-16T09:59:04,707 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731751137553.f3ed23ac403772e0fe4ca9ee892544ec., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 , the old value is region=TestAcidGuarantees,,1731751137553.f3ed23ac403772e0fe4ca9ee892544ec., hostname=037c58a19ddb,37267,1731751077844, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3ed23ac403772e0fe4ca9ee892544ec, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:59:04,707 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731751137553.f3ed23ac403772e0fe4ca9ee892544ec., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3ed23ac403772e0fe4ca9ee892544ec, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:59:04,707 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731751137553.f3ed23ac403772e0fe4ca9ee892544ec., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 because the exception is null or not the one we care about 2024-11-16T09:59:04,715 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3ed23ac403772e0fe4ca9ee892544ec, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T09:59:04,715 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34368 deadline: 1731751204713, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3ed23ac403772e0fe4ca9ee892544ec, server=037c58a19ddb,37267,1731751077844 2024-11-16T09:59:04,715 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731751137553.f3ed23ac403772e0fe4ca9ee892544ec., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 , the old value is region=TestAcidGuarantees,,1731751137553.f3ed23ac403772e0fe4ca9ee892544ec., hostname=037c58a19ddb,37267,1731751077844, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3ed23ac403772e0fe4ca9ee892544ec, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:59:04,715 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731751137553.f3ed23ac403772e0fe4ca9ee892544ec., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3ed23ac403772e0fe4ca9ee892544ec, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:59:04,715 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731751137553.f3ed23ac403772e0fe4ca9ee892544ec., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 because the exception is null or not the one we care about 2024-11-16T09:59:04,792 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=37267 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=67 2024-11-16T09:59:04,792 DEBUG [RS_FLUSH_OPERATIONS-regionserver/037c58a19ddb:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731751137553.f3ed23ac403772e0fe4ca9ee892544ec. 2024-11-16T09:59:04,792 DEBUG [RS_FLUSH_OPERATIONS-regionserver/037c58a19ddb:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731751137553.f3ed23ac403772e0fe4ca9ee892544ec. as already flushing 2024-11-16T09:59:04,792 DEBUG [RS_FLUSH_OPERATIONS-regionserver/037c58a19ddb:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731751137553.f3ed23ac403772e0fe4ca9ee892544ec. 2024-11-16T09:59:04,792 ERROR [RS_FLUSH_OPERATIONS-regionserver/037c58a19ddb:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] handler.RSProcedureHandler(58): pid=67 java.io.IOException: Unable to complete flush {ENCODED => f3ed23ac403772e0fe4ca9ee892544ec, NAME => 'TestAcidGuarantees,,1731751137553.f3ed23ac403772e0fe4ca9ee892544ec.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T09:59:04,792 DEBUG [RS_FLUSH_OPERATIONS-regionserver/037c58a19ddb:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=67 java.io.IOException: Unable to complete flush {ENCODED => f3ed23ac403772e0fe4ca9ee892544ec, NAME => 'TestAcidGuarantees,,1731751137553.f3ed23ac403772e0fe4ca9ee892544ec.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T09:59:04,793 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37877 {}] master.HMaster(4177): Remote procedure failed, pid=67 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => f3ed23ac403772e0fe4ca9ee892544ec, NAME => 'TestAcidGuarantees,,1731751137553.f3ed23ac403772e0fe4ca9ee892544ec.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => f3ed23ac403772e0fe4ca9ee892544ec, NAME => 'TestAcidGuarantees,,1731751137553.f3ed23ac403772e0fe4ca9ee892544ec.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T09:59:04,803 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37877 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=66 2024-11-16T09:59:04,892 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=53.67 KB at sequenceid=116 (bloomFilter=true), to=hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/f3ed23ac403772e0fe4ca9ee892544ec/.tmp/B/0500168b3f534c849c2134d112232df9 2024-11-16T09:59:04,900 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/f3ed23ac403772e0fe4ca9ee892544ec/.tmp/C/1003ab34c2204b9da43a22b69b58c635 is 50, key is test_row_0/C:col10/1731751143951/Put/seqid=0 2024-11-16T09:59:04,904 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37591 is added to blk_1073742099_1275 (size=12001) 2024-11-16T09:59:04,945 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=37267 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=67 2024-11-16T09:59:04,945 DEBUG [RS_FLUSH_OPERATIONS-regionserver/037c58a19ddb:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731751137553.f3ed23ac403772e0fe4ca9ee892544ec. 2024-11-16T09:59:04,945 DEBUG [RS_FLUSH_OPERATIONS-regionserver/037c58a19ddb:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731751137553.f3ed23ac403772e0fe4ca9ee892544ec. as already flushing 2024-11-16T09:59:04,945 DEBUG [RS_FLUSH_OPERATIONS-regionserver/037c58a19ddb:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731751137553.f3ed23ac403772e0fe4ca9ee892544ec. 2024-11-16T09:59:04,945 ERROR [RS_FLUSH_OPERATIONS-regionserver/037c58a19ddb:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] handler.RSProcedureHandler(58): pid=67 java.io.IOException: Unable to complete flush {ENCODED => f3ed23ac403772e0fe4ca9ee892544ec, NAME => 'TestAcidGuarantees,,1731751137553.f3ed23ac403772e0fe4ca9ee892544ec.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T09:59:04,946 DEBUG [RS_FLUSH_OPERATIONS-regionserver/037c58a19ddb:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=67 java.io.IOException: Unable to complete flush {ENCODED => f3ed23ac403772e0fe4ca9ee892544ec, NAME => 'TestAcidGuarantees,,1731751137553.f3ed23ac403772e0fe4ca9ee892544ec.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T09:59:04,946 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37877 {}] master.HMaster(4177): Remote procedure failed, pid=67 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => f3ed23ac403772e0fe4ca9ee892544ec, NAME => 'TestAcidGuarantees,,1731751137553.f3ed23ac403772e0fe4ca9ee892544ec.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => f3ed23ac403772e0fe4ca9ee892544ec, NAME => 'TestAcidGuarantees,,1731751137553.f3ed23ac403772e0fe4ca9ee892544ec.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T09:59:05,098 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=37267 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=67 2024-11-16T09:59:05,098 DEBUG [RS_FLUSH_OPERATIONS-regionserver/037c58a19ddb:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731751137553.f3ed23ac403772e0fe4ca9ee892544ec. 2024-11-16T09:59:05,099 DEBUG [RS_FLUSH_OPERATIONS-regionserver/037c58a19ddb:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731751137553.f3ed23ac403772e0fe4ca9ee892544ec. as already flushing 2024-11-16T09:59:05,099 DEBUG [RS_FLUSH_OPERATIONS-regionserver/037c58a19ddb:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731751137553.f3ed23ac403772e0fe4ca9ee892544ec. 2024-11-16T09:59:05,099 ERROR [RS_FLUSH_OPERATIONS-regionserver/037c58a19ddb:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] handler.RSProcedureHandler(58): pid=67 java.io.IOException: Unable to complete flush {ENCODED => f3ed23ac403772e0fe4ca9ee892544ec, NAME => 'TestAcidGuarantees,,1731751137553.f3ed23ac403772e0fe4ca9ee892544ec.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T09:59:05,099 DEBUG [RS_FLUSH_OPERATIONS-regionserver/037c58a19ddb:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=67 java.io.IOException: Unable to complete flush {ENCODED => f3ed23ac403772e0fe4ca9ee892544ec, NAME => 'TestAcidGuarantees,,1731751137553.f3ed23ac403772e0fe4ca9ee892544ec.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T09:59:05,099 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37877 {}] master.HMaster(4177): Remote procedure failed, pid=67 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => f3ed23ac403772e0fe4ca9ee892544ec, NAME => 'TestAcidGuarantees,,1731751137553.f3ed23ac403772e0fe4ca9ee892544ec.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => f3ed23ac403772e0fe4ca9ee892544ec, NAME => 'TestAcidGuarantees,,1731751137553.f3ed23ac403772e0fe4ca9ee892544ec.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T09:59:05,215 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3ed23ac403772e0fe4ca9ee892544ec, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T09:59:05,216 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34386 deadline: 1731751205212, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3ed23ac403772e0fe4ca9ee892544ec, server=037c58a19ddb,37267,1731751077844 2024-11-16T09:59:05,216 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3ed23ac403772e0fe4ca9ee892544ec, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T09:59:05,216 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34408 deadline: 1731751205213, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3ed23ac403772e0fe4ca9ee892544ec, server=037c58a19ddb,37267,1731751077844 2024-11-16T09:59:05,216 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3ed23ac403772e0fe4ca9ee892544ec, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T09:59:05,216 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731751137553.f3ed23ac403772e0fe4ca9ee892544ec., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 , the old value is region=TestAcidGuarantees,,1731751137553.f3ed23ac403772e0fe4ca9ee892544ec., hostname=037c58a19ddb,37267,1731751077844, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3ed23ac403772e0fe4ca9ee892544ec, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:59:05,216 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34414 deadline: 1731751205213, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3ed23ac403772e0fe4ca9ee892544ec, server=037c58a19ddb,37267,1731751077844 2024-11-16T09:59:05,216 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731751137553.f3ed23ac403772e0fe4ca9ee892544ec., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3ed23ac403772e0fe4ca9ee892544ec, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:59:05,216 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731751137553.f3ed23ac403772e0fe4ca9ee892544ec., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 because the exception is null or not the one we care about 2024-11-16T09:59:05,216 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3ed23ac403772e0fe4ca9ee892544ec, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T09:59:05,216 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731751137553.f3ed23ac403772e0fe4ca9ee892544ec., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 , the old value is region=TestAcidGuarantees,,1731751137553.f3ed23ac403772e0fe4ca9ee892544ec., hostname=037c58a19ddb,37267,1731751077844, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3ed23ac403772e0fe4ca9ee892544ec, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:59:05,216 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731751137553.f3ed23ac403772e0fe4ca9ee892544ec., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3ed23ac403772e0fe4ca9ee892544ec, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:59:05,216 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731751137553.f3ed23ac403772e0fe4ca9ee892544ec., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 because the exception is null or not the one we care about 2024-11-16T09:59:05,216 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34382 deadline: 1731751205213, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3ed23ac403772e0fe4ca9ee892544ec, server=037c58a19ddb,37267,1731751077844 2024-11-16T09:59:05,216 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731751137553.f3ed23ac403772e0fe4ca9ee892544ec., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 , the old value is region=TestAcidGuarantees,,1731751137553.f3ed23ac403772e0fe4ca9ee892544ec., hostname=037c58a19ddb,37267,1731751077844, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3ed23ac403772e0fe4ca9ee892544ec, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:59:05,216 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731751137553.f3ed23ac403772e0fe4ca9ee892544ec., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3ed23ac403772e0fe4ca9ee892544ec, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:59:05,216 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731751137553.f3ed23ac403772e0fe4ca9ee892544ec., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 because the exception is null or not the one we care about 2024-11-16T09:59:05,216 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731751137553.f3ed23ac403772e0fe4ca9ee892544ec., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 , the old value is region=TestAcidGuarantees,,1731751137553.f3ed23ac403772e0fe4ca9ee892544ec., hostname=037c58a19ddb,37267,1731751077844, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3ed23ac403772e0fe4ca9ee892544ec, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:59:05,216 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731751137553.f3ed23ac403772e0fe4ca9ee892544ec., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3ed23ac403772e0fe4ca9ee892544ec, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:59:05,216 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731751137553.f3ed23ac403772e0fe4ca9ee892544ec., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 because the exception is null or not the one we care about 2024-11-16T09:59:05,223 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3ed23ac403772e0fe4ca9ee892544ec, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T09:59:05,223 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34368 deadline: 1731751205223, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3ed23ac403772e0fe4ca9ee892544ec, server=037c58a19ddb,37267,1731751077844 2024-11-16T09:59:05,224 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731751137553.f3ed23ac403772e0fe4ca9ee892544ec., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 , the old value is region=TestAcidGuarantees,,1731751137553.f3ed23ac403772e0fe4ca9ee892544ec., hostname=037c58a19ddb,37267,1731751077844, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3ed23ac403772e0fe4ca9ee892544ec, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:59:05,224 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731751137553.f3ed23ac403772e0fe4ca9ee892544ec., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3ed23ac403772e0fe4ca9ee892544ec, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:59:05,224 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731751137553.f3ed23ac403772e0fe4ca9ee892544ec., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 because the exception is null or not the one we care about 2024-11-16T09:59:05,252 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=37267 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=67 2024-11-16T09:59:05,252 DEBUG [RS_FLUSH_OPERATIONS-regionserver/037c58a19ddb:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731751137553.f3ed23ac403772e0fe4ca9ee892544ec. 2024-11-16T09:59:05,252 DEBUG [RS_FLUSH_OPERATIONS-regionserver/037c58a19ddb:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731751137553.f3ed23ac403772e0fe4ca9ee892544ec. as already flushing 2024-11-16T09:59:05,252 DEBUG [RS_FLUSH_OPERATIONS-regionserver/037c58a19ddb:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731751137553.f3ed23ac403772e0fe4ca9ee892544ec. 2024-11-16T09:59:05,252 ERROR [RS_FLUSH_OPERATIONS-regionserver/037c58a19ddb:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] handler.RSProcedureHandler(58): pid=67 java.io.IOException: Unable to complete flush {ENCODED => f3ed23ac403772e0fe4ca9ee892544ec, NAME => 'TestAcidGuarantees,,1731751137553.f3ed23ac403772e0fe4ca9ee892544ec.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T09:59:05,252 DEBUG [RS_FLUSH_OPERATIONS-regionserver/037c58a19ddb:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=67 java.io.IOException: Unable to complete flush {ENCODED => f3ed23ac403772e0fe4ca9ee892544ec, NAME => 'TestAcidGuarantees,,1731751137553.f3ed23ac403772e0fe4ca9ee892544ec.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T09:59:05,253 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37877 {}] master.HMaster(4177): Remote procedure failed, pid=67 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => f3ed23ac403772e0fe4ca9ee892544ec, NAME => 'TestAcidGuarantees,,1731751137553.f3ed23ac403772e0fe4ca9ee892544ec.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => f3ed23ac403772e0fe4ca9ee892544ec, NAME => 'TestAcidGuarantees,,1731751137553.f3ed23ac403772e0fe4ca9ee892544ec.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T09:59:05,305 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=53.67 KB at sequenceid=116 (bloomFilter=true), to=hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/f3ed23ac403772e0fe4ca9ee892544ec/.tmp/C/1003ab34c2204b9da43a22b69b58c635 2024-11-16T09:59:05,310 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/f3ed23ac403772e0fe4ca9ee892544ec/.tmp/A/27f0ac8785f94a86aedae7fb6146d397 as hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/f3ed23ac403772e0fe4ca9ee892544ec/A/27f0ac8785f94a86aedae7fb6146d397 2024-11-16T09:59:05,313 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37877 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=66 2024-11-16T09:59:05,314 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/f3ed23ac403772e0fe4ca9ee892544ec/A/27f0ac8785f94a86aedae7fb6146d397, entries=150, sequenceid=116, filesize=11.7 K 2024-11-16T09:59:05,315 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/f3ed23ac403772e0fe4ca9ee892544ec/.tmp/B/0500168b3f534c849c2134d112232df9 as hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/f3ed23ac403772e0fe4ca9ee892544ec/B/0500168b3f534c849c2134d112232df9 2024-11-16T09:59:05,318 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/f3ed23ac403772e0fe4ca9ee892544ec/B/0500168b3f534c849c2134d112232df9, entries=150, sequenceid=116, filesize=11.7 K 2024-11-16T09:59:05,319 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/f3ed23ac403772e0fe4ca9ee892544ec/.tmp/C/1003ab34c2204b9da43a22b69b58c635 as hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/f3ed23ac403772e0fe4ca9ee892544ec/C/1003ab34c2204b9da43a22b69b58c635 2024-11-16T09:59:05,323 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/f3ed23ac403772e0fe4ca9ee892544ec/C/1003ab34c2204b9da43a22b69b58c635, entries=150, sequenceid=116, filesize=11.7 K 2024-11-16T09:59:05,324 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~161.02 KB/164880, heapSize ~422.58 KB/432720, currentSize=53.67 KB/54960 for f3ed23ac403772e0fe4ca9ee892544ec in 1258ms, sequenceid=116, compaction requested=true 2024-11-16T09:59:05,324 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for f3ed23ac403772e0fe4ca9ee892544ec: 2024-11-16T09:59:05,324 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store f3ed23ac403772e0fe4ca9ee892544ec:A, priority=-2147483648, current under compaction store size is 1 2024-11-16T09:59:05,325 DEBUG [RS:0;037c58a19ddb:37267-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-16T09:59:05,325 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-16T09:59:05,325 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store f3ed23ac403772e0fe4ca9ee892544ec:B, priority=-2147483648, current under compaction store size is 2 2024-11-16T09:59:05,325 DEBUG [RS:0;037c58a19ddb:37267-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-16T09:59:05,325 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-16T09:59:05,325 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store f3ed23ac403772e0fe4ca9ee892544ec:C, priority=-2147483648, current under compaction store size is 3 2024-11-16T09:59:05,325 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-16T09:59:05,326 DEBUG [RS:0;037c58a19ddb:37267-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36141 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-16T09:59:05,326 DEBUG [RS:0;037c58a19ddb:37267-shortCompactions-0 {}] regionserver.HStore(1541): f3ed23ac403772e0fe4ca9ee892544ec/A is initiating minor compaction (all files) 2024-11-16T09:59:05,326 INFO [RS:0;037c58a19ddb:37267-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of f3ed23ac403772e0fe4ca9ee892544ec/A in TestAcidGuarantees,,1731751137553.f3ed23ac403772e0fe4ca9ee892544ec. 2024-11-16T09:59:05,326 INFO [RS:0;037c58a19ddb:37267-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/f3ed23ac403772e0fe4ca9ee892544ec/A/c4ef90a8928e4ab49e0b5f591292b9b9, hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/f3ed23ac403772e0fe4ca9ee892544ec/A/699d58ef21354836b7b3485f7bcfb2c0, hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/f3ed23ac403772e0fe4ca9ee892544ec/A/27f0ac8785f94a86aedae7fb6146d397] into tmpdir=hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/f3ed23ac403772e0fe4ca9ee892544ec/.tmp, totalSize=35.3 K 2024-11-16T09:59:05,326 DEBUG [RS:0;037c58a19ddb:37267-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36141 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-16T09:59:05,326 DEBUG [RS:0;037c58a19ddb:37267-longCompactions-0 {}] regionserver.HStore(1541): f3ed23ac403772e0fe4ca9ee892544ec/B is initiating minor compaction (all files) 2024-11-16T09:59:05,326 INFO [RS:0;037c58a19ddb:37267-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of f3ed23ac403772e0fe4ca9ee892544ec/B in TestAcidGuarantees,,1731751137553.f3ed23ac403772e0fe4ca9ee892544ec. 2024-11-16T09:59:05,327 INFO [RS:0;037c58a19ddb:37267-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/f3ed23ac403772e0fe4ca9ee892544ec/B/1ab050ebce744bc993da534cbe3617aa, hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/f3ed23ac403772e0fe4ca9ee892544ec/B/2801d6506c4844e8ab66e1d8707f85ba, hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/f3ed23ac403772e0fe4ca9ee892544ec/B/0500168b3f534c849c2134d112232df9] into tmpdir=hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/f3ed23ac403772e0fe4ca9ee892544ec/.tmp, totalSize=35.3 K 2024-11-16T09:59:05,327 DEBUG [RS:0;037c58a19ddb:37267-shortCompactions-0 {}] compactions.Compactor(225): Compacting c4ef90a8928e4ab49e0b5f591292b9b9, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=74, earliestPutTs=1731751141108 2024-11-16T09:59:05,327 DEBUG [RS:0;037c58a19ddb:37267-longCompactions-0 {}] compactions.Compactor(225): Compacting 1ab050ebce744bc993da534cbe3617aa, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=74, earliestPutTs=1731751141108 2024-11-16T09:59:05,327 DEBUG [RS:0;037c58a19ddb:37267-shortCompactions-0 {}] compactions.Compactor(225): Compacting 699d58ef21354836b7b3485f7bcfb2c0, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=86, earliestPutTs=1731751141747 2024-11-16T09:59:05,328 DEBUG [RS:0;037c58a19ddb:37267-longCompactions-0 {}] compactions.Compactor(225): Compacting 2801d6506c4844e8ab66e1d8707f85ba, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=86, earliestPutTs=1731751141747 2024-11-16T09:59:05,328 DEBUG [RS:0;037c58a19ddb:37267-shortCompactions-0 {}] compactions.Compactor(225): Compacting 27f0ac8785f94a86aedae7fb6146d397, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=116, earliestPutTs=1731751143951 2024-11-16T09:59:05,328 DEBUG [RS:0;037c58a19ddb:37267-longCompactions-0 {}] compactions.Compactor(225): Compacting 0500168b3f534c849c2134d112232df9, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=116, earliestPutTs=1731751143951 2024-11-16T09:59:05,338 INFO [RS:0;037c58a19ddb:37267-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): f3ed23ac403772e0fe4ca9ee892544ec#A#compaction#231 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-16T09:59:05,338 DEBUG [RS:0;037c58a19ddb:37267-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/f3ed23ac403772e0fe4ca9ee892544ec/.tmp/A/fc6aac9ad19e442cbbab9d08b96d6884 is 50, key is test_row_0/A:col10/1731751143951/Put/seqid=0 2024-11-16T09:59:05,350 INFO [RS:0;037c58a19ddb:37267-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): f3ed23ac403772e0fe4ca9ee892544ec#B#compaction#232 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-16T09:59:05,350 DEBUG [RS:0;037c58a19ddb:37267-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/f3ed23ac403772e0fe4ca9ee892544ec/.tmp/B/e2460513c96644a9b59d2108e16c30c1 is 50, key is test_row_0/B:col10/1731751143951/Put/seqid=0 2024-11-16T09:59:05,355 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37591 is added to blk_1073742100_1276 (size=12241) 2024-11-16T09:59:05,377 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37591 is added to blk_1073742101_1277 (size=12241) 2024-11-16T09:59:05,383 DEBUG [RS:0;037c58a19ddb:37267-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/f3ed23ac403772e0fe4ca9ee892544ec/.tmp/B/e2460513c96644a9b59d2108e16c30c1 as hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/f3ed23ac403772e0fe4ca9ee892544ec/B/e2460513c96644a9b59d2108e16c30c1 2024-11-16T09:59:05,388 INFO [RS:0;037c58a19ddb:37267-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in f3ed23ac403772e0fe4ca9ee892544ec/B of f3ed23ac403772e0fe4ca9ee892544ec into e2460513c96644a9b59d2108e16c30c1(size=12.0 K), total size for store is 12.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-16T09:59:05,388 DEBUG [RS:0;037c58a19ddb:37267-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for f3ed23ac403772e0fe4ca9ee892544ec: 2024-11-16T09:59:05,388 INFO [RS:0;037c58a19ddb:37267-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731751137553.f3ed23ac403772e0fe4ca9ee892544ec., storeName=f3ed23ac403772e0fe4ca9ee892544ec/B, priority=13, startTime=1731751145325; duration=0sec 2024-11-16T09:59:05,388 DEBUG [RS:0;037c58a19ddb:37267-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-16T09:59:05,388 DEBUG [RS:0;037c58a19ddb:37267-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: f3ed23ac403772e0fe4ca9ee892544ec:B 2024-11-16T09:59:05,388 DEBUG [RS:0;037c58a19ddb:37267-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-16T09:59:05,390 DEBUG [RS:0;037c58a19ddb:37267-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36141 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-16T09:59:05,390 DEBUG [RS:0;037c58a19ddb:37267-longCompactions-0 {}] regionserver.HStore(1541): f3ed23ac403772e0fe4ca9ee892544ec/C is initiating minor compaction (all files) 2024-11-16T09:59:05,390 INFO [RS:0;037c58a19ddb:37267-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of f3ed23ac403772e0fe4ca9ee892544ec/C in TestAcidGuarantees,,1731751137553.f3ed23ac403772e0fe4ca9ee892544ec. 2024-11-16T09:59:05,391 INFO [RS:0;037c58a19ddb:37267-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/f3ed23ac403772e0fe4ca9ee892544ec/C/4336fc254c134e0b93fa1380900fa45f, hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/f3ed23ac403772e0fe4ca9ee892544ec/C/7a98e6e6c80d4413a5a5f621e7789567, hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/f3ed23ac403772e0fe4ca9ee892544ec/C/1003ab34c2204b9da43a22b69b58c635] into tmpdir=hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/f3ed23ac403772e0fe4ca9ee892544ec/.tmp, totalSize=35.3 K 2024-11-16T09:59:05,391 DEBUG [RS:0;037c58a19ddb:37267-longCompactions-0 {}] compactions.Compactor(225): Compacting 4336fc254c134e0b93fa1380900fa45f, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=74, earliestPutTs=1731751141108 2024-11-16T09:59:05,391 DEBUG [RS:0;037c58a19ddb:37267-longCompactions-0 {}] compactions.Compactor(225): Compacting 7a98e6e6c80d4413a5a5f621e7789567, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=86, earliestPutTs=1731751141747 2024-11-16T09:59:05,392 DEBUG [RS:0;037c58a19ddb:37267-longCompactions-0 {}] compactions.Compactor(225): Compacting 1003ab34c2204b9da43a22b69b58c635, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=116, earliestPutTs=1731751143951 2024-11-16T09:59:05,398 INFO [RS:0;037c58a19ddb:37267-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): f3ed23ac403772e0fe4ca9ee892544ec#C#compaction#233 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-16T09:59:05,398 DEBUG [RS:0;037c58a19ddb:37267-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/f3ed23ac403772e0fe4ca9ee892544ec/.tmp/C/3a6d4b5ddb9a453da1011ee0b36a82b7 is 50, key is test_row_0/C:col10/1731751143951/Put/seqid=0 2024-11-16T09:59:05,402 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37591 is added to blk_1073742102_1278 (size=12241) 2024-11-16T09:59:05,406 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=37267 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=67 2024-11-16T09:59:05,406 DEBUG [RS_FLUSH_OPERATIONS-regionserver/037c58a19ddb:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731751137553.f3ed23ac403772e0fe4ca9ee892544ec. 2024-11-16T09:59:05,406 INFO [RS_FLUSH_OPERATIONS-regionserver/037c58a19ddb:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HRegion(2902): Flushing f3ed23ac403772e0fe4ca9ee892544ec 3/3 column families, dataSize=53.67 KB heapSize=141.38 KB 2024-11-16T09:59:05,411 DEBUG [RS_FLUSH_OPERATIONS-regionserver/037c58a19ddb:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/f3ed23ac403772e0fe4ca9ee892544ec/.tmp/A/ae5eee1b468f4af7a61d7367b1991b0b is 50, key is test_row_0/A:col10/1731751144074/Put/seqid=0 2024-11-16T09:59:05,413 DEBUG [RS:0;037c58a19ddb:37267-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/f3ed23ac403772e0fe4ca9ee892544ec/.tmp/C/3a6d4b5ddb9a453da1011ee0b36a82b7 as hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/f3ed23ac403772e0fe4ca9ee892544ec/C/3a6d4b5ddb9a453da1011ee0b36a82b7 2024-11-16T09:59:05,417 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37591 is added to blk_1073742103_1279 (size=9657) 2024-11-16T09:59:05,421 INFO [RS:0;037c58a19ddb:37267-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in f3ed23ac403772e0fe4ca9ee892544ec/C of f3ed23ac403772e0fe4ca9ee892544ec into 3a6d4b5ddb9a453da1011ee0b36a82b7(size=12.0 K), total size for store is 12.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-16T09:59:05,421 DEBUG [RS:0;037c58a19ddb:37267-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for f3ed23ac403772e0fe4ca9ee892544ec: 2024-11-16T09:59:05,421 INFO [RS:0;037c58a19ddb:37267-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731751137553.f3ed23ac403772e0fe4ca9ee892544ec., storeName=f3ed23ac403772e0fe4ca9ee892544ec/C, priority=13, startTime=1731751145325; duration=0sec 2024-11-16T09:59:05,421 DEBUG [RS:0;037c58a19ddb:37267-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-16T09:59:05,421 DEBUG [RS:0;037c58a19ddb:37267-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: f3ed23ac403772e0fe4ca9ee892544ec:C 2024-11-16T09:59:05,762 DEBUG [RS:0;037c58a19ddb:37267-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/f3ed23ac403772e0fe4ca9ee892544ec/.tmp/A/fc6aac9ad19e442cbbab9d08b96d6884 as hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/f3ed23ac403772e0fe4ca9ee892544ec/A/fc6aac9ad19e442cbbab9d08b96d6884 2024-11-16T09:59:05,768 INFO [RS:0;037c58a19ddb:37267-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in f3ed23ac403772e0fe4ca9ee892544ec/A of f3ed23ac403772e0fe4ca9ee892544ec into fc6aac9ad19e442cbbab9d08b96d6884(size=12.0 K), total size for store is 12.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-16T09:59:05,768 DEBUG [RS:0;037c58a19ddb:37267-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for f3ed23ac403772e0fe4ca9ee892544ec: 2024-11-16T09:59:05,769 INFO [RS:0;037c58a19ddb:37267-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731751137553.f3ed23ac403772e0fe4ca9ee892544ec., storeName=f3ed23ac403772e0fe4ca9ee892544ec/A, priority=13, startTime=1731751145324; duration=0sec 2024-11-16T09:59:05,769 DEBUG [RS:0;037c58a19ddb:37267-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-16T09:59:05,769 DEBUG [RS:0;037c58a19ddb:37267-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: f3ed23ac403772e0fe4ca9ee892544ec:A 2024-11-16T09:59:05,818 INFO [RS_FLUSH_OPERATIONS-regionserver/037c58a19ddb:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=128 (bloomFilter=true), to=hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/f3ed23ac403772e0fe4ca9ee892544ec/.tmp/A/ae5eee1b468f4af7a61d7367b1991b0b 2024-11-16T09:59:05,826 DEBUG [RS_FLUSH_OPERATIONS-regionserver/037c58a19ddb:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/f3ed23ac403772e0fe4ca9ee892544ec/.tmp/B/8cb982fa7b484471b2ab01d4d21174f0 is 50, key is test_row_0/B:col10/1731751144074/Put/seqid=0 2024-11-16T09:59:05,830 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37591 is added to blk_1073742104_1280 (size=9657) 2024-11-16T09:59:06,225 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] regionserver.HRegion(8855): Flush requested on f3ed23ac403772e0fe4ca9ee892544ec 2024-11-16T09:59:06,225 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731751137553.f3ed23ac403772e0fe4ca9ee892544ec. as already flushing 2024-11-16T09:59:06,231 INFO [RS_FLUSH_OPERATIONS-regionserver/037c58a19ddb:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=128 (bloomFilter=true), to=hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/f3ed23ac403772e0fe4ca9ee892544ec/.tmp/B/8cb982fa7b484471b2ab01d4d21174f0 2024-11-16T09:59:06,240 DEBUG [RS_FLUSH_OPERATIONS-regionserver/037c58a19ddb:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/f3ed23ac403772e0fe4ca9ee892544ec/.tmp/C/3349459d23bd43dd85817bbd6c9715fd is 50, key is test_row_0/C:col10/1731751144074/Put/seqid=0 2024-11-16T09:59:06,245 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37591 is added to blk_1073742105_1281 (size=9657) 2024-11-16T09:59:06,255 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3ed23ac403772e0fe4ca9ee892544ec, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T09:59:06,256 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34368 deadline: 1731751206250, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3ed23ac403772e0fe4ca9ee892544ec, server=037c58a19ddb,37267,1731751077844 2024-11-16T09:59:06,256 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3ed23ac403772e0fe4ca9ee892544ec, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T09:59:06,256 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731751137553.f3ed23ac403772e0fe4ca9ee892544ec., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 , the old value is region=TestAcidGuarantees,,1731751137553.f3ed23ac403772e0fe4ca9ee892544ec., hostname=037c58a19ddb,37267,1731751077844, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3ed23ac403772e0fe4ca9ee892544ec, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:59:06,256 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731751137553.f3ed23ac403772e0fe4ca9ee892544ec., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3ed23ac403772e0fe4ca9ee892544ec, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:59:06,256 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34386 deadline: 1731751206251, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3ed23ac403772e0fe4ca9ee892544ec, server=037c58a19ddb,37267,1731751077844 2024-11-16T09:59:06,256 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731751137553.f3ed23ac403772e0fe4ca9ee892544ec., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 because the exception is null or not the one we care about 2024-11-16T09:59:06,256 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3ed23ac403772e0fe4ca9ee892544ec, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T09:59:06,256 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731751137553.f3ed23ac403772e0fe4ca9ee892544ec., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 , the old value is region=TestAcidGuarantees,,1731751137553.f3ed23ac403772e0fe4ca9ee892544ec., hostname=037c58a19ddb,37267,1731751077844, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3ed23ac403772e0fe4ca9ee892544ec, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:59:06,256 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731751137553.f3ed23ac403772e0fe4ca9ee892544ec., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3ed23ac403772e0fe4ca9ee892544ec, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:59:06,256 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34414 deadline: 1731751206252, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3ed23ac403772e0fe4ca9ee892544ec, server=037c58a19ddb,37267,1731751077844 2024-11-16T09:59:06,256 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731751137553.f3ed23ac403772e0fe4ca9ee892544ec., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 because the exception is null or not the one we care about 2024-11-16T09:59:06,257 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731751137553.f3ed23ac403772e0fe4ca9ee892544ec., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 , the old value is region=TestAcidGuarantees,,1731751137553.f3ed23ac403772e0fe4ca9ee892544ec., hostname=037c58a19ddb,37267,1731751077844, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3ed23ac403772e0fe4ca9ee892544ec, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:59:06,257 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731751137553.f3ed23ac403772e0fe4ca9ee892544ec., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3ed23ac403772e0fe4ca9ee892544ec, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:59:06,257 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3ed23ac403772e0fe4ca9ee892544ec, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T09:59:06,257 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731751137553.f3ed23ac403772e0fe4ca9ee892544ec., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 because the exception is null or not the one we care about 2024-11-16T09:59:06,257 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34382 deadline: 1731751206254, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3ed23ac403772e0fe4ca9ee892544ec, server=037c58a19ddb,37267,1731751077844 2024-11-16T09:59:06,257 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731751137553.f3ed23ac403772e0fe4ca9ee892544ec., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 , the old value is region=TestAcidGuarantees,,1731751137553.f3ed23ac403772e0fe4ca9ee892544ec., hostname=037c58a19ddb,37267,1731751077844, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3ed23ac403772e0fe4ca9ee892544ec, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:59:06,257 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731751137553.f3ed23ac403772e0fe4ca9ee892544ec., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3ed23ac403772e0fe4ca9ee892544ec, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:59:06,257 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731751137553.f3ed23ac403772e0fe4ca9ee892544ec., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 because the exception is null or not the one we care about 2024-11-16T09:59:06,259 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3ed23ac403772e0fe4ca9ee892544ec, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T09:59:06,259 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34408 deadline: 1731751206256, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3ed23ac403772e0fe4ca9ee892544ec, server=037c58a19ddb,37267,1731751077844 2024-11-16T09:59:06,260 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731751137553.f3ed23ac403772e0fe4ca9ee892544ec., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 , the old value is region=TestAcidGuarantees,,1731751137553.f3ed23ac403772e0fe4ca9ee892544ec., hostname=037c58a19ddb,37267,1731751077844, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3ed23ac403772e0fe4ca9ee892544ec, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:59:06,260 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731751137553.f3ed23ac403772e0fe4ca9ee892544ec., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3ed23ac403772e0fe4ca9ee892544ec, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:59:06,260 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731751137553.f3ed23ac403772e0fe4ca9ee892544ec., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 because the exception is null or not the one we care about 2024-11-16T09:59:06,323 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37877 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=66 2024-11-16T09:59:06,364 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3ed23ac403772e0fe4ca9ee892544ec, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T09:59:06,364 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34368 deadline: 1731751206363, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3ed23ac403772e0fe4ca9ee892544ec, server=037c58a19ddb,37267,1731751077844 2024-11-16T09:59:06,364 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731751137553.f3ed23ac403772e0fe4ca9ee892544ec., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 , the old value is region=TestAcidGuarantees,,1731751137553.f3ed23ac403772e0fe4ca9ee892544ec., hostname=037c58a19ddb,37267,1731751077844, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3ed23ac403772e0fe4ca9ee892544ec, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:59:06,365 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731751137553.f3ed23ac403772e0fe4ca9ee892544ec., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3ed23ac403772e0fe4ca9ee892544ec, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:59:06,365 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731751137553.f3ed23ac403772e0fe4ca9ee892544ec., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 because the exception is null or not the one we care about 2024-11-16T09:59:06,365 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3ed23ac403772e0fe4ca9ee892544ec, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T09:59:06,365 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34386 deadline: 1731751206363, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3ed23ac403772e0fe4ca9ee892544ec, server=037c58a19ddb,37267,1731751077844 2024-11-16T09:59:06,365 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731751137553.f3ed23ac403772e0fe4ca9ee892544ec., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 , the old value is region=TestAcidGuarantees,,1731751137553.f3ed23ac403772e0fe4ca9ee892544ec., hostname=037c58a19ddb,37267,1731751077844, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3ed23ac403772e0fe4ca9ee892544ec, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:59:06,365 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3ed23ac403772e0fe4ca9ee892544ec, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T09:59:06,365 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731751137553.f3ed23ac403772e0fe4ca9ee892544ec., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3ed23ac403772e0fe4ca9ee892544ec, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:59:06,365 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731751137553.f3ed23ac403772e0fe4ca9ee892544ec., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 because the exception is null or not the one we care about 2024-11-16T09:59:06,365 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34414 deadline: 1731751206363, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3ed23ac403772e0fe4ca9ee892544ec, server=037c58a19ddb,37267,1731751077844 2024-11-16T09:59:06,365 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731751137553.f3ed23ac403772e0fe4ca9ee892544ec., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 , the old value is region=TestAcidGuarantees,,1731751137553.f3ed23ac403772e0fe4ca9ee892544ec., hostname=037c58a19ddb,37267,1731751077844, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3ed23ac403772e0fe4ca9ee892544ec, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:59:06,365 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3ed23ac403772e0fe4ca9ee892544ec, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T09:59:06,365 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731751137553.f3ed23ac403772e0fe4ca9ee892544ec., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3ed23ac403772e0fe4ca9ee892544ec, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:59:06,365 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731751137553.f3ed23ac403772e0fe4ca9ee892544ec., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 because the exception is null or not the one we care about 2024-11-16T09:59:06,365 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34382 deadline: 1731751206363, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3ed23ac403772e0fe4ca9ee892544ec, server=037c58a19ddb,37267,1731751077844 2024-11-16T09:59:06,365 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3ed23ac403772e0fe4ca9ee892544ec, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T09:59:06,366 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34408 deadline: 1731751206363, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3ed23ac403772e0fe4ca9ee892544ec, server=037c58a19ddb,37267,1731751077844 2024-11-16T09:59:06,366 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731751137553.f3ed23ac403772e0fe4ca9ee892544ec., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 , the old value is region=TestAcidGuarantees,,1731751137553.f3ed23ac403772e0fe4ca9ee892544ec., hostname=037c58a19ddb,37267,1731751077844, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3ed23ac403772e0fe4ca9ee892544ec, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:59:06,366 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731751137553.f3ed23ac403772e0fe4ca9ee892544ec., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3ed23ac403772e0fe4ca9ee892544ec, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:59:06,366 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731751137553.f3ed23ac403772e0fe4ca9ee892544ec., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 because the exception is null or not the one we care about 2024-11-16T09:59:06,366 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731751137553.f3ed23ac403772e0fe4ca9ee892544ec., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 , the old value is region=TestAcidGuarantees,,1731751137553.f3ed23ac403772e0fe4ca9ee892544ec., hostname=037c58a19ddb,37267,1731751077844, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3ed23ac403772e0fe4ca9ee892544ec, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:59:06,366 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731751137553.f3ed23ac403772e0fe4ca9ee892544ec., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3ed23ac403772e0fe4ca9ee892544ec, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:59:06,366 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731751137553.f3ed23ac403772e0fe4ca9ee892544ec., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 because the exception is null or not the one we care about 2024-11-16T09:59:06,574 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3ed23ac403772e0fe4ca9ee892544ec, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T09:59:06,574 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34368 deadline: 1731751206573, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3ed23ac403772e0fe4ca9ee892544ec, server=037c58a19ddb,37267,1731751077844 2024-11-16T09:59:06,575 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3ed23ac403772e0fe4ca9ee892544ec, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T09:59:06,575 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3ed23ac403772e0fe4ca9ee892544ec, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T09:59:06,575 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34414 deadline: 1731751206573, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3ed23ac403772e0fe4ca9ee892544ec, server=037c58a19ddb,37267,1731751077844 2024-11-16T09:59:06,575 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731751137553.f3ed23ac403772e0fe4ca9ee892544ec., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 , the old value is region=TestAcidGuarantees,,1731751137553.f3ed23ac403772e0fe4ca9ee892544ec., hostname=037c58a19ddb,37267,1731751077844, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3ed23ac403772e0fe4ca9ee892544ec, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:59:06,575 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731751137553.f3ed23ac403772e0fe4ca9ee892544ec., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3ed23ac403772e0fe4ca9ee892544ec, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:59:06,575 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34386 deadline: 1731751206573, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3ed23ac403772e0fe4ca9ee892544ec, server=037c58a19ddb,37267,1731751077844 2024-11-16T09:59:06,575 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731751137553.f3ed23ac403772e0fe4ca9ee892544ec., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 because the exception is null or not the one we care about 2024-11-16T09:59:06,575 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731751137553.f3ed23ac403772e0fe4ca9ee892544ec., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 , the old value is region=TestAcidGuarantees,,1731751137553.f3ed23ac403772e0fe4ca9ee892544ec., hostname=037c58a19ddb,37267,1731751077844, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3ed23ac403772e0fe4ca9ee892544ec, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:59:06,575 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731751137553.f3ed23ac403772e0fe4ca9ee892544ec., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 , the old value is region=TestAcidGuarantees,,1731751137553.f3ed23ac403772e0fe4ca9ee892544ec., hostname=037c58a19ddb,37267,1731751077844, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3ed23ac403772e0fe4ca9ee892544ec, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:59:06,575 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3ed23ac403772e0fe4ca9ee892544ec, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T09:59:06,575 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731751137553.f3ed23ac403772e0fe4ca9ee892544ec., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3ed23ac403772e0fe4ca9ee892544ec, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:59:06,575 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731751137553.f3ed23ac403772e0fe4ca9ee892544ec., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3ed23ac403772e0fe4ca9ee892544ec, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:59:06,575 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731751137553.f3ed23ac403772e0fe4ca9ee892544ec., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 because the exception is null or not the one we care about 2024-11-16T09:59:06,575 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3ed23ac403772e0fe4ca9ee892544ec, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T09:59:06,575 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731751137553.f3ed23ac403772e0fe4ca9ee892544ec., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 because the exception is null or not the one we care about 2024-11-16T09:59:06,575 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34382 deadline: 1731751206573, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3ed23ac403772e0fe4ca9ee892544ec, server=037c58a19ddb,37267,1731751077844 2024-11-16T09:59:06,575 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34408 deadline: 1731751206573, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3ed23ac403772e0fe4ca9ee892544ec, server=037c58a19ddb,37267,1731751077844 2024-11-16T09:59:06,575 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731751137553.f3ed23ac403772e0fe4ca9ee892544ec., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 , the old value is region=TestAcidGuarantees,,1731751137553.f3ed23ac403772e0fe4ca9ee892544ec., hostname=037c58a19ddb,37267,1731751077844, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3ed23ac403772e0fe4ca9ee892544ec, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:59:06,575 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731751137553.f3ed23ac403772e0fe4ca9ee892544ec., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 , the old value is region=TestAcidGuarantees,,1731751137553.f3ed23ac403772e0fe4ca9ee892544ec., hostname=037c58a19ddb,37267,1731751077844, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3ed23ac403772e0fe4ca9ee892544ec, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:59:06,575 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731751137553.f3ed23ac403772e0fe4ca9ee892544ec., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3ed23ac403772e0fe4ca9ee892544ec, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:59:06,575 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731751137553.f3ed23ac403772e0fe4ca9ee892544ec., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 because the exception is null or not the one we care about 2024-11-16T09:59:06,575 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731751137553.f3ed23ac403772e0fe4ca9ee892544ec., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3ed23ac403772e0fe4ca9ee892544ec, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:59:06,575 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731751137553.f3ed23ac403772e0fe4ca9ee892544ec., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 because the exception is null or not the one we care about 2024-11-16T09:59:06,646 INFO [RS_FLUSH_OPERATIONS-regionserver/037c58a19ddb:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=128 (bloomFilter=true), to=hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/f3ed23ac403772e0fe4ca9ee892544ec/.tmp/C/3349459d23bd43dd85817bbd6c9715fd 2024-11-16T09:59:06,651 DEBUG [RS_FLUSH_OPERATIONS-regionserver/037c58a19ddb:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/f3ed23ac403772e0fe4ca9ee892544ec/.tmp/A/ae5eee1b468f4af7a61d7367b1991b0b as hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/f3ed23ac403772e0fe4ca9ee892544ec/A/ae5eee1b468f4af7a61d7367b1991b0b 2024-11-16T09:59:06,655 INFO [RS_FLUSH_OPERATIONS-regionserver/037c58a19ddb:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/f3ed23ac403772e0fe4ca9ee892544ec/A/ae5eee1b468f4af7a61d7367b1991b0b, entries=100, sequenceid=128, filesize=9.4 K 2024-11-16T09:59:06,657 DEBUG [RS_FLUSH_OPERATIONS-regionserver/037c58a19ddb:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/f3ed23ac403772e0fe4ca9ee892544ec/.tmp/B/8cb982fa7b484471b2ab01d4d21174f0 as hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/f3ed23ac403772e0fe4ca9ee892544ec/B/8cb982fa7b484471b2ab01d4d21174f0 2024-11-16T09:59:06,661 INFO [RS_FLUSH_OPERATIONS-regionserver/037c58a19ddb:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/f3ed23ac403772e0fe4ca9ee892544ec/B/8cb982fa7b484471b2ab01d4d21174f0, entries=100, sequenceid=128, filesize=9.4 K 2024-11-16T09:59:06,662 DEBUG [RS_FLUSH_OPERATIONS-regionserver/037c58a19ddb:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/f3ed23ac403772e0fe4ca9ee892544ec/.tmp/C/3349459d23bd43dd85817bbd6c9715fd as hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/f3ed23ac403772e0fe4ca9ee892544ec/C/3349459d23bd43dd85817bbd6c9715fd 2024-11-16T09:59:06,667 INFO [RS_FLUSH_OPERATIONS-regionserver/037c58a19ddb:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/f3ed23ac403772e0fe4ca9ee892544ec/C/3349459d23bd43dd85817bbd6c9715fd, entries=100, sequenceid=128, filesize=9.4 K 2024-11-16T09:59:06,668 INFO [RS_FLUSH_OPERATIONS-regionserver/037c58a19ddb:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~141.33 KB/144720, currentSize=147.60 KB/151140 for f3ed23ac403772e0fe4ca9ee892544ec in 1262ms, sequenceid=128, compaction requested=false 2024-11-16T09:59:06,668 DEBUG [RS_FLUSH_OPERATIONS-regionserver/037c58a19ddb:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HRegion(2603): Flush status journal for f3ed23ac403772e0fe4ca9ee892544ec: 2024-11-16T09:59:06,668 DEBUG [RS_FLUSH_OPERATIONS-regionserver/037c58a19ddb:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731751137553.f3ed23ac403772e0fe4ca9ee892544ec. 2024-11-16T09:59:06,668 DEBUG [RS_FLUSH_OPERATIONS-regionserver/037c58a19ddb:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=67 2024-11-16T09:59:06,668 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37877 {}] master.HMaster(4169): Remote procedure done, pid=67 2024-11-16T09:59:06,671 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=67, resume processing ppid=66 2024-11-16T09:59:06,671 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=67, ppid=66, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.4920 sec 2024-11-16T09:59:06,673 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=66, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=66, table=TestAcidGuarantees in 2.4980 sec 2024-11-16T09:59:06,885 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] regionserver.HRegion(8855): Flush requested on f3ed23ac403772e0fe4ca9ee892544ec 2024-11-16T09:59:06,885 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-16T09:59:06,886 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing f3ed23ac403772e0fe4ca9ee892544ec 3/3 column families, dataSize=154.31 KB heapSize=405.05 KB 2024-11-16T09:59:06,892 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/f3ed23ac403772e0fe4ca9ee892544ec/.tmp/A/d241f3b4a9f54d6eaf9c03b0d64cdbb6 is 50, key is test_row_0/A:col10/1731751146885/Put/seqid=0 2024-11-16T09:59:06,892 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3ed23ac403772e0fe4ca9ee892544ec, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T09:59:06,892 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3ed23ac403772e0fe4ca9ee892544ec, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T09:59:06,892 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34408 deadline: 1731751206888, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3ed23ac403772e0fe4ca9ee892544ec, server=037c58a19ddb,37267,1731751077844 2024-11-16T09:59:06,892 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34382 deadline: 1731751206887, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3ed23ac403772e0fe4ca9ee892544ec, server=037c58a19ddb,37267,1731751077844 2024-11-16T09:59:06,892 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3ed23ac403772e0fe4ca9ee892544ec, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T09:59:06,893 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731751137553.f3ed23ac403772e0fe4ca9ee892544ec., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 , the old value is region=TestAcidGuarantees,,1731751137553.f3ed23ac403772e0fe4ca9ee892544ec., hostname=037c58a19ddb,37267,1731751077844, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3ed23ac403772e0fe4ca9ee892544ec, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:59:06,893 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731751137553.f3ed23ac403772e0fe4ca9ee892544ec., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 , the old value is region=TestAcidGuarantees,,1731751137553.f3ed23ac403772e0fe4ca9ee892544ec., hostname=037c58a19ddb,37267,1731751077844, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3ed23ac403772e0fe4ca9ee892544ec, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:59:06,893 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731751137553.f3ed23ac403772e0fe4ca9ee892544ec., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3ed23ac403772e0fe4ca9ee892544ec, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:59:06,893 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731751137553.f3ed23ac403772e0fe4ca9ee892544ec., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3ed23ac403772e0fe4ca9ee892544ec, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:59:06,893 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34368 deadline: 1731751206890, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3ed23ac403772e0fe4ca9ee892544ec, server=037c58a19ddb,37267,1731751077844 2024-11-16T09:59:06,893 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731751137553.f3ed23ac403772e0fe4ca9ee892544ec., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 because the exception is null or not the one we care about 2024-11-16T09:59:06,893 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731751137553.f3ed23ac403772e0fe4ca9ee892544ec., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 because the exception is null or not the one we care about 2024-11-16T09:59:06,893 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731751137553.f3ed23ac403772e0fe4ca9ee892544ec., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 , the old value is region=TestAcidGuarantees,,1731751137553.f3ed23ac403772e0fe4ca9ee892544ec., hostname=037c58a19ddb,37267,1731751077844, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3ed23ac403772e0fe4ca9ee892544ec, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:59:06,893 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731751137553.f3ed23ac403772e0fe4ca9ee892544ec., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3ed23ac403772e0fe4ca9ee892544ec, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:59:06,893 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731751137553.f3ed23ac403772e0fe4ca9ee892544ec., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 because the exception is null or not the one we care about 2024-11-16T09:59:06,895 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37591 is added to blk_1073742106_1282 (size=12151) 2024-11-16T09:59:06,895 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3ed23ac403772e0fe4ca9ee892544ec, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T09:59:06,896 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34414 deadline: 1731751206892, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3ed23ac403772e0fe4ca9ee892544ec, server=037c58a19ddb,37267,1731751077844 2024-11-16T09:59:06,896 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3ed23ac403772e0fe4ca9ee892544ec, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T09:59:06,896 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731751137553.f3ed23ac403772e0fe4ca9ee892544ec., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 , the old value is region=TestAcidGuarantees,,1731751137553.f3ed23ac403772e0fe4ca9ee892544ec., hostname=037c58a19ddb,37267,1731751077844, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3ed23ac403772e0fe4ca9ee892544ec, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:59:06,896 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731751137553.f3ed23ac403772e0fe4ca9ee892544ec., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3ed23ac403772e0fe4ca9ee892544ec, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:59:06,896 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731751137553.f3ed23ac403772e0fe4ca9ee892544ec., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 because the exception is null or not the one we care about 2024-11-16T09:59:06,896 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34386 deadline: 1731751206892, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3ed23ac403772e0fe4ca9ee892544ec, server=037c58a19ddb,37267,1731751077844 2024-11-16T09:59:06,896 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731751137553.f3ed23ac403772e0fe4ca9ee892544ec., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 , the old value is region=TestAcidGuarantees,,1731751137553.f3ed23ac403772e0fe4ca9ee892544ec., hostname=037c58a19ddb,37267,1731751077844, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3ed23ac403772e0fe4ca9ee892544ec, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:59:06,896 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731751137553.f3ed23ac403772e0fe4ca9ee892544ec., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3ed23ac403772e0fe4ca9ee892544ec, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:59:06,896 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731751137553.f3ed23ac403772e0fe4ca9ee892544ec., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 because the exception is null or not the one we care about 2024-11-16T09:59:06,897 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=58.14 KB at sequenceid=159 (bloomFilter=true), to=hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/f3ed23ac403772e0fe4ca9ee892544ec/.tmp/A/d241f3b4a9f54d6eaf9c03b0d64cdbb6 2024-11-16T09:59:06,905 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/f3ed23ac403772e0fe4ca9ee892544ec/.tmp/B/f889bfe6d6574832a2b04ec61ff06197 is 50, key is test_row_0/B:col10/1731751146885/Put/seqid=0 2024-11-16T09:59:06,914 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37591 is added to blk_1073742107_1283 (size=12151) 2024-11-16T09:59:07,005 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3ed23ac403772e0fe4ca9ee892544ec, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T09:59:07,005 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34408 deadline: 1731751207003, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3ed23ac403772e0fe4ca9ee892544ec, server=037c58a19ddb,37267,1731751077844 2024-11-16T09:59:07,005 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3ed23ac403772e0fe4ca9ee892544ec, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T09:59:07,005 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731751137553.f3ed23ac403772e0fe4ca9ee892544ec., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 , the old value is region=TestAcidGuarantees,,1731751137553.f3ed23ac403772e0fe4ca9ee892544ec., hostname=037c58a19ddb,37267,1731751077844, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3ed23ac403772e0fe4ca9ee892544ec, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:59:07,006 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731751137553.f3ed23ac403772e0fe4ca9ee892544ec., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3ed23ac403772e0fe4ca9ee892544ec, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:59:07,006 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731751137553.f3ed23ac403772e0fe4ca9ee892544ec., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 because the exception is null or not the one we care about 2024-11-16T09:59:07,006 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34368 deadline: 1731751207003, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3ed23ac403772e0fe4ca9ee892544ec, server=037c58a19ddb,37267,1731751077844 2024-11-16T09:59:07,006 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731751137553.f3ed23ac403772e0fe4ca9ee892544ec., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 , the old value is region=TestAcidGuarantees,,1731751137553.f3ed23ac403772e0fe4ca9ee892544ec., hostname=037c58a19ddb,37267,1731751077844, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3ed23ac403772e0fe4ca9ee892544ec, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:59:07,006 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731751137553.f3ed23ac403772e0fe4ca9ee892544ec., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3ed23ac403772e0fe4ca9ee892544ec, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:59:07,006 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731751137553.f3ed23ac403772e0fe4ca9ee892544ec., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 because the exception is null or not the one we care about 2024-11-16T09:59:07,006 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3ed23ac403772e0fe4ca9ee892544ec, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T09:59:07,006 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34382 deadline: 1731751207003, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3ed23ac403772e0fe4ca9ee892544ec, server=037c58a19ddb,37267,1731751077844 2024-11-16T09:59:07,006 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3ed23ac403772e0fe4ca9ee892544ec, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T09:59:07,006 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34414 deadline: 1731751207003, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3ed23ac403772e0fe4ca9ee892544ec, server=037c58a19ddb,37267,1731751077844 2024-11-16T09:59:07,006 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3ed23ac403772e0fe4ca9ee892544ec, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T09:59:07,006 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731751137553.f3ed23ac403772e0fe4ca9ee892544ec., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 , the old value is region=TestAcidGuarantees,,1731751137553.f3ed23ac403772e0fe4ca9ee892544ec., hostname=037c58a19ddb,37267,1731751077844, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3ed23ac403772e0fe4ca9ee892544ec, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:59:07,006 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731751137553.f3ed23ac403772e0fe4ca9ee892544ec., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3ed23ac403772e0fe4ca9ee892544ec, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:59:07,006 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34386 deadline: 1731751207003, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3ed23ac403772e0fe4ca9ee892544ec, server=037c58a19ddb,37267,1731751077844 2024-11-16T09:59:07,006 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731751137553.f3ed23ac403772e0fe4ca9ee892544ec., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 because the exception is null or not the one we care about 2024-11-16T09:59:07,006 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731751137553.f3ed23ac403772e0fe4ca9ee892544ec., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 , the old value is region=TestAcidGuarantees,,1731751137553.f3ed23ac403772e0fe4ca9ee892544ec., hostname=037c58a19ddb,37267,1731751077844, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3ed23ac403772e0fe4ca9ee892544ec, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:59:07,006 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731751137553.f3ed23ac403772e0fe4ca9ee892544ec., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3ed23ac403772e0fe4ca9ee892544ec, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:59:07,006 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731751137553.f3ed23ac403772e0fe4ca9ee892544ec., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 because the exception is null or not the one we care about 2024-11-16T09:59:07,007 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731751137553.f3ed23ac403772e0fe4ca9ee892544ec., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 , the old value is region=TestAcidGuarantees,,1731751137553.f3ed23ac403772e0fe4ca9ee892544ec., hostname=037c58a19ddb,37267,1731751077844, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3ed23ac403772e0fe4ca9ee892544ec, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:59:07,007 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731751137553.f3ed23ac403772e0fe4ca9ee892544ec., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3ed23ac403772e0fe4ca9ee892544ec, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:59:07,007 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731751137553.f3ed23ac403772e0fe4ca9ee892544ec., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 because the exception is null or not the one we care about 2024-11-16T09:59:07,215 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3ed23ac403772e0fe4ca9ee892544ec, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T09:59:07,215 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34408 deadline: 1731751207213, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3ed23ac403772e0fe4ca9ee892544ec, server=037c58a19ddb,37267,1731751077844 2024-11-16T09:59:07,215 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731751137553.f3ed23ac403772e0fe4ca9ee892544ec., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 , the old value is region=TestAcidGuarantees,,1731751137553.f3ed23ac403772e0fe4ca9ee892544ec., hostname=037c58a19ddb,37267,1731751077844, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3ed23ac403772e0fe4ca9ee892544ec, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:59:07,216 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731751137553.f3ed23ac403772e0fe4ca9ee892544ec., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3ed23ac403772e0fe4ca9ee892544ec, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:59:07,216 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731751137553.f3ed23ac403772e0fe4ca9ee892544ec., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 because the exception is null or not the one we care about 2024-11-16T09:59:07,216 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3ed23ac403772e0fe4ca9ee892544ec, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T09:59:07,216 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34368 deadline: 1731751207213, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3ed23ac403772e0fe4ca9ee892544ec, server=037c58a19ddb,37267,1731751077844 2024-11-16T09:59:07,216 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731751137553.f3ed23ac403772e0fe4ca9ee892544ec., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 , the old value is region=TestAcidGuarantees,,1731751137553.f3ed23ac403772e0fe4ca9ee892544ec., hostname=037c58a19ddb,37267,1731751077844, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3ed23ac403772e0fe4ca9ee892544ec, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:59:07,216 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3ed23ac403772e0fe4ca9ee892544ec, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T09:59:07,216 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731751137553.f3ed23ac403772e0fe4ca9ee892544ec., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3ed23ac403772e0fe4ca9ee892544ec, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:59:07,216 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731751137553.f3ed23ac403772e0fe4ca9ee892544ec., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 because the exception is null or not the one we care about 2024-11-16T09:59:07,216 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34382 deadline: 1731751207213, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3ed23ac403772e0fe4ca9ee892544ec, server=037c58a19ddb,37267,1731751077844 2024-11-16T09:59:07,217 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731751137553.f3ed23ac403772e0fe4ca9ee892544ec., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 , the old value is region=TestAcidGuarantees,,1731751137553.f3ed23ac403772e0fe4ca9ee892544ec., hostname=037c58a19ddb,37267,1731751077844, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3ed23ac403772e0fe4ca9ee892544ec, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:59:07,217 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731751137553.f3ed23ac403772e0fe4ca9ee892544ec., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3ed23ac403772e0fe4ca9ee892544ec, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:59:07,217 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3ed23ac403772e0fe4ca9ee892544ec, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T09:59:07,217 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731751137553.f3ed23ac403772e0fe4ca9ee892544ec., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 because the exception is null or not the one we care about 2024-11-16T09:59:07,217 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34414 deadline: 1731751207213, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3ed23ac403772e0fe4ca9ee892544ec, server=037c58a19ddb,37267,1731751077844 2024-11-16T09:59:07,217 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3ed23ac403772e0fe4ca9ee892544ec, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T09:59:07,217 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34386 deadline: 1731751207213, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3ed23ac403772e0fe4ca9ee892544ec, server=037c58a19ddb,37267,1731751077844 2024-11-16T09:59:07,217 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731751137553.f3ed23ac403772e0fe4ca9ee892544ec., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 , the old value is region=TestAcidGuarantees,,1731751137553.f3ed23ac403772e0fe4ca9ee892544ec., hostname=037c58a19ddb,37267,1731751077844, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3ed23ac403772e0fe4ca9ee892544ec, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:59:07,217 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731751137553.f3ed23ac403772e0fe4ca9ee892544ec., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3ed23ac403772e0fe4ca9ee892544ec, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:59:07,217 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731751137553.f3ed23ac403772e0fe4ca9ee892544ec., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 because the exception is null or not the one we care about 2024-11-16T09:59:07,217 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731751137553.f3ed23ac403772e0fe4ca9ee892544ec., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 , the old value is region=TestAcidGuarantees,,1731751137553.f3ed23ac403772e0fe4ca9ee892544ec., hostname=037c58a19ddb,37267,1731751077844, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3ed23ac403772e0fe4ca9ee892544ec, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:59:07,217 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731751137553.f3ed23ac403772e0fe4ca9ee892544ec., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3ed23ac403772e0fe4ca9ee892544ec, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:59:07,217 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731751137553.f3ed23ac403772e0fe4ca9ee892544ec., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 because the exception is null or not the one we care about 2024-11-16T09:59:07,315 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=58.14 KB at sequenceid=159 (bloomFilter=true), to=hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/f3ed23ac403772e0fe4ca9ee892544ec/.tmp/B/f889bfe6d6574832a2b04ec61ff06197 2024-11-16T09:59:07,323 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/f3ed23ac403772e0fe4ca9ee892544ec/.tmp/C/c828236206584489b0b4b191772c331b is 50, key is test_row_0/C:col10/1731751146885/Put/seqid=0 2024-11-16T09:59:07,327 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37591 is added to blk_1073742108_1284 (size=12151) 2024-11-16T09:59:07,524 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3ed23ac403772e0fe4ca9ee892544ec, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T09:59:07,525 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34408 deadline: 1731751207522, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3ed23ac403772e0fe4ca9ee892544ec, server=037c58a19ddb,37267,1731751077844 2024-11-16T09:59:07,525 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3ed23ac403772e0fe4ca9ee892544ec, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T09:59:07,525 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34368 deadline: 1731751207523, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3ed23ac403772e0fe4ca9ee892544ec, server=037c58a19ddb,37267,1731751077844 2024-11-16T09:59:07,525 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731751137553.f3ed23ac403772e0fe4ca9ee892544ec., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 , the old value is region=TestAcidGuarantees,,1731751137553.f3ed23ac403772e0fe4ca9ee892544ec., hostname=037c58a19ddb,37267,1731751077844, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3ed23ac403772e0fe4ca9ee892544ec, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:59:07,525 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731751137553.f3ed23ac403772e0fe4ca9ee892544ec., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3ed23ac403772e0fe4ca9ee892544ec, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:59:07,525 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731751137553.f3ed23ac403772e0fe4ca9ee892544ec., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 because the exception is null or not the one we care about 2024-11-16T09:59:07,525 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731751137553.f3ed23ac403772e0fe4ca9ee892544ec., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 , the old value is region=TestAcidGuarantees,,1731751137553.f3ed23ac403772e0fe4ca9ee892544ec., hostname=037c58a19ddb,37267,1731751077844, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3ed23ac403772e0fe4ca9ee892544ec, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:59:07,525 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3ed23ac403772e0fe4ca9ee892544ec, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T09:59:07,525 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731751137553.f3ed23ac403772e0fe4ca9ee892544ec., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3ed23ac403772e0fe4ca9ee892544ec, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:59:07,525 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731751137553.f3ed23ac403772e0fe4ca9ee892544ec., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 because the exception is null or not the one we care about 2024-11-16T09:59:07,525 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34382 deadline: 1731751207523, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3ed23ac403772e0fe4ca9ee892544ec, server=037c58a19ddb,37267,1731751077844 2024-11-16T09:59:07,525 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731751137553.f3ed23ac403772e0fe4ca9ee892544ec., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 , the old value is region=TestAcidGuarantees,,1731751137553.f3ed23ac403772e0fe4ca9ee892544ec., hostname=037c58a19ddb,37267,1731751077844, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3ed23ac403772e0fe4ca9ee892544ec, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:59:07,525 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731751137553.f3ed23ac403772e0fe4ca9ee892544ec., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3ed23ac403772e0fe4ca9ee892544ec, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:59:07,525 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3ed23ac403772e0fe4ca9ee892544ec, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T09:59:07,525 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731751137553.f3ed23ac403772e0fe4ca9ee892544ec., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 because the exception is null or not the one we care about 2024-11-16T09:59:07,526 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34414 deadline: 1731751207523, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3ed23ac403772e0fe4ca9ee892544ec, server=037c58a19ddb,37267,1731751077844 2024-11-16T09:59:07,526 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3ed23ac403772e0fe4ca9ee892544ec, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T09:59:07,526 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34386 deadline: 1731751207523, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3ed23ac403772e0fe4ca9ee892544ec, server=037c58a19ddb,37267,1731751077844 2024-11-16T09:59:07,526 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731751137553.f3ed23ac403772e0fe4ca9ee892544ec., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 , the old value is region=TestAcidGuarantees,,1731751137553.f3ed23ac403772e0fe4ca9ee892544ec., hostname=037c58a19ddb,37267,1731751077844, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3ed23ac403772e0fe4ca9ee892544ec, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:59:07,526 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731751137553.f3ed23ac403772e0fe4ca9ee892544ec., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3ed23ac403772e0fe4ca9ee892544ec, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:59:07,526 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731751137553.f3ed23ac403772e0fe4ca9ee892544ec., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 because the exception is null or not the one we care about 2024-11-16T09:59:07,526 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731751137553.f3ed23ac403772e0fe4ca9ee892544ec., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 , the old value is region=TestAcidGuarantees,,1731751137553.f3ed23ac403772e0fe4ca9ee892544ec., hostname=037c58a19ddb,37267,1731751077844, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3ed23ac403772e0fe4ca9ee892544ec, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:59:07,526 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731751137553.f3ed23ac403772e0fe4ca9ee892544ec., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3ed23ac403772e0fe4ca9ee892544ec, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:59:07,526 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731751137553.f3ed23ac403772e0fe4ca9ee892544ec., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 because the exception is null or not the one we care about 2024-11-16T09:59:07,539 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_TestAcidGuarantees 2024-11-16T09:59:07,539 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_default_table_TestAcidGuarantees Metrics about Tables on a single HBase RegionServer 2024-11-16T09:59:07,727 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=58.14 KB at sequenceid=159 (bloomFilter=true), to=hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/f3ed23ac403772e0fe4ca9ee892544ec/.tmp/C/c828236206584489b0b4b191772c331b 2024-11-16T09:59:07,734 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/f3ed23ac403772e0fe4ca9ee892544ec/.tmp/A/d241f3b4a9f54d6eaf9c03b0d64cdbb6 as hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/f3ed23ac403772e0fe4ca9ee892544ec/A/d241f3b4a9f54d6eaf9c03b0d64cdbb6 2024-11-16T09:59:07,738 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/f3ed23ac403772e0fe4ca9ee892544ec/A/d241f3b4a9f54d6eaf9c03b0d64cdbb6, entries=150, sequenceid=159, filesize=11.9 K 2024-11-16T09:59:07,739 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/f3ed23ac403772e0fe4ca9ee892544ec/.tmp/B/f889bfe6d6574832a2b04ec61ff06197 as hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/f3ed23ac403772e0fe4ca9ee892544ec/B/f889bfe6d6574832a2b04ec61ff06197 2024-11-16T09:59:07,743 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/f3ed23ac403772e0fe4ca9ee892544ec/B/f889bfe6d6574832a2b04ec61ff06197, entries=150, sequenceid=159, filesize=11.9 K 2024-11-16T09:59:07,744 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/f3ed23ac403772e0fe4ca9ee892544ec/.tmp/C/c828236206584489b0b4b191772c331b as hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/f3ed23ac403772e0fe4ca9ee892544ec/C/c828236206584489b0b4b191772c331b 2024-11-16T09:59:07,748 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/f3ed23ac403772e0fe4ca9ee892544ec/C/c828236206584489b0b4b191772c331b, entries=150, sequenceid=159, filesize=11.9 K 2024-11-16T09:59:07,749 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~174.43 KB/178620, heapSize ~457.73 KB/468720, currentSize=26.84 KB/27480 for f3ed23ac403772e0fe4ca9ee892544ec in 864ms, sequenceid=159, compaction requested=true 2024-11-16T09:59:07,749 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for f3ed23ac403772e0fe4ca9ee892544ec: 2024-11-16T09:59:07,749 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store f3ed23ac403772e0fe4ca9ee892544ec:A, priority=-2147483648, current under compaction store size is 1 2024-11-16T09:59:07,749 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-16T09:59:07,749 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store f3ed23ac403772e0fe4ca9ee892544ec:B, priority=-2147483648, current under compaction store size is 2 2024-11-16T09:59:07,749 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-16T09:59:07,749 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store f3ed23ac403772e0fe4ca9ee892544ec:C, priority=-2147483648, current under compaction store size is 3 2024-11-16T09:59:07,749 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=2), splitQueue=0 2024-11-16T09:59:07,749 DEBUG [RS:0;037c58a19ddb:37267-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-16T09:59:07,750 DEBUG [RS:0;037c58a19ddb:37267-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-16T09:59:07,750 DEBUG [RS:0;037c58a19ddb:37267-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 34049 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-16T09:59:07,750 DEBUG [RS:0;037c58a19ddb:37267-shortCompactions-0 {}] regionserver.HStore(1541): f3ed23ac403772e0fe4ca9ee892544ec/A is initiating minor compaction (all files) 2024-11-16T09:59:07,750 DEBUG [RS:0;037c58a19ddb:37267-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 34049 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-16T09:59:07,750 INFO [RS:0;037c58a19ddb:37267-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of f3ed23ac403772e0fe4ca9ee892544ec/A in TestAcidGuarantees,,1731751137553.f3ed23ac403772e0fe4ca9ee892544ec. 2024-11-16T09:59:07,750 DEBUG [RS:0;037c58a19ddb:37267-longCompactions-0 {}] regionserver.HStore(1541): f3ed23ac403772e0fe4ca9ee892544ec/B is initiating minor compaction (all files) 2024-11-16T09:59:07,750 INFO [RS:0;037c58a19ddb:37267-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/f3ed23ac403772e0fe4ca9ee892544ec/A/fc6aac9ad19e442cbbab9d08b96d6884, hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/f3ed23ac403772e0fe4ca9ee892544ec/A/ae5eee1b468f4af7a61d7367b1991b0b, hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/f3ed23ac403772e0fe4ca9ee892544ec/A/d241f3b4a9f54d6eaf9c03b0d64cdbb6] into tmpdir=hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/f3ed23ac403772e0fe4ca9ee892544ec/.tmp, totalSize=33.3 K 2024-11-16T09:59:07,751 INFO [RS:0;037c58a19ddb:37267-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of f3ed23ac403772e0fe4ca9ee892544ec/B in TestAcidGuarantees,,1731751137553.f3ed23ac403772e0fe4ca9ee892544ec. 2024-11-16T09:59:07,751 INFO [RS:0;037c58a19ddb:37267-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/f3ed23ac403772e0fe4ca9ee892544ec/B/e2460513c96644a9b59d2108e16c30c1, hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/f3ed23ac403772e0fe4ca9ee892544ec/B/8cb982fa7b484471b2ab01d4d21174f0, hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/f3ed23ac403772e0fe4ca9ee892544ec/B/f889bfe6d6574832a2b04ec61ff06197] into tmpdir=hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/f3ed23ac403772e0fe4ca9ee892544ec/.tmp, totalSize=33.3 K 2024-11-16T09:59:07,751 DEBUG [RS:0;037c58a19ddb:37267-shortCompactions-0 {}] compactions.Compactor(225): Compacting fc6aac9ad19e442cbbab9d08b96d6884, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=116, earliestPutTs=1731751143951 2024-11-16T09:59:07,751 DEBUG [RS:0;037c58a19ddb:37267-longCompactions-0 {}] compactions.Compactor(225): Compacting e2460513c96644a9b59d2108e16c30c1, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=116, earliestPutTs=1731751143951 2024-11-16T09:59:07,751 DEBUG [RS:0;037c58a19ddb:37267-shortCompactions-0 {}] compactions.Compactor(225): Compacting ae5eee1b468f4af7a61d7367b1991b0b, keycount=100, bloomtype=ROW, size=9.4 K, encoding=NONE, compression=NONE, seqNum=128, earliestPutTs=1731751144074 2024-11-16T09:59:07,751 DEBUG [RS:0;037c58a19ddb:37267-longCompactions-0 {}] compactions.Compactor(225): Compacting 8cb982fa7b484471b2ab01d4d21174f0, keycount=100, bloomtype=ROW, size=9.4 K, encoding=NONE, compression=NONE, seqNum=128, earliestPutTs=1731751144074 2024-11-16T09:59:07,752 DEBUG [RS:0;037c58a19ddb:37267-shortCompactions-0 {}] compactions.Compactor(225): Compacting d241f3b4a9f54d6eaf9c03b0d64cdbb6, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=159, earliestPutTs=1731751146251 2024-11-16T09:59:07,752 DEBUG [RS:0;037c58a19ddb:37267-longCompactions-0 {}] compactions.Compactor(225): Compacting f889bfe6d6574832a2b04ec61ff06197, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=159, earliestPutTs=1731751146251 2024-11-16T09:59:07,766 INFO [RS:0;037c58a19ddb:37267-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): f3ed23ac403772e0fe4ca9ee892544ec#B#compaction#240 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-16T09:59:07,767 DEBUG [RS:0;037c58a19ddb:37267-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/f3ed23ac403772e0fe4ca9ee892544ec/.tmp/B/59e77647903f4960ababcd7f53bcaae7 is 50, key is test_row_0/B:col10/1731751146885/Put/seqid=0 2024-11-16T09:59:07,773 INFO [RS:0;037c58a19ddb:37267-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): f3ed23ac403772e0fe4ca9ee892544ec#A#compaction#241 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-16T09:59:07,774 DEBUG [RS:0;037c58a19ddb:37267-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/f3ed23ac403772e0fe4ca9ee892544ec/.tmp/A/f9b5d8015d1b4e76a61f9ad1279908b4 is 50, key is test_row_0/A:col10/1731751146885/Put/seqid=0 2024-11-16T09:59:07,779 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37591 is added to blk_1073742109_1285 (size=12493) 2024-11-16T09:59:07,779 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37591 is added to blk_1073742110_1286 (size=12493) 2024-11-16T09:59:07,785 DEBUG [RS:0;037c58a19ddb:37267-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/f3ed23ac403772e0fe4ca9ee892544ec/.tmp/A/f9b5d8015d1b4e76a61f9ad1279908b4 as hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/f3ed23ac403772e0fe4ca9ee892544ec/A/f9b5d8015d1b4e76a61f9ad1279908b4 2024-11-16T09:59:07,792 INFO [RS:0;037c58a19ddb:37267-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in f3ed23ac403772e0fe4ca9ee892544ec/A of f3ed23ac403772e0fe4ca9ee892544ec into f9b5d8015d1b4e76a61f9ad1279908b4(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-16T09:59:07,792 DEBUG [RS:0;037c58a19ddb:37267-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for f3ed23ac403772e0fe4ca9ee892544ec: 2024-11-16T09:59:07,792 INFO [RS:0;037c58a19ddb:37267-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731751137553.f3ed23ac403772e0fe4ca9ee892544ec., storeName=f3ed23ac403772e0fe4ca9ee892544ec/A, priority=13, startTime=1731751147749; duration=0sec 2024-11-16T09:59:07,792 DEBUG [RS:0;037c58a19ddb:37267-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-16T09:59:07,792 DEBUG [RS:0;037c58a19ddb:37267-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: f3ed23ac403772e0fe4ca9ee892544ec:A 2024-11-16T09:59:07,792 DEBUG [RS:0;037c58a19ddb:37267-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-16T09:59:07,793 DEBUG [RS:0;037c58a19ddb:37267-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 34049 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-16T09:59:07,793 DEBUG [RS:0;037c58a19ddb:37267-shortCompactions-0 {}] regionserver.HStore(1541): f3ed23ac403772e0fe4ca9ee892544ec/C is initiating minor compaction (all files) 2024-11-16T09:59:07,793 INFO [RS:0;037c58a19ddb:37267-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of f3ed23ac403772e0fe4ca9ee892544ec/C in TestAcidGuarantees,,1731751137553.f3ed23ac403772e0fe4ca9ee892544ec. 2024-11-16T09:59:07,794 INFO [RS:0;037c58a19ddb:37267-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/f3ed23ac403772e0fe4ca9ee892544ec/C/3a6d4b5ddb9a453da1011ee0b36a82b7, hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/f3ed23ac403772e0fe4ca9ee892544ec/C/3349459d23bd43dd85817bbd6c9715fd, hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/f3ed23ac403772e0fe4ca9ee892544ec/C/c828236206584489b0b4b191772c331b] into tmpdir=hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/f3ed23ac403772e0fe4ca9ee892544ec/.tmp, totalSize=33.3 K 2024-11-16T09:59:07,794 DEBUG [RS:0;037c58a19ddb:37267-shortCompactions-0 {}] compactions.Compactor(225): Compacting 3a6d4b5ddb9a453da1011ee0b36a82b7, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=116, earliestPutTs=1731751143951 2024-11-16T09:59:07,794 DEBUG [RS:0;037c58a19ddb:37267-shortCompactions-0 {}] compactions.Compactor(225): Compacting 3349459d23bd43dd85817bbd6c9715fd, keycount=100, bloomtype=ROW, size=9.4 K, encoding=NONE, compression=NONE, seqNum=128, earliestPutTs=1731751144074 2024-11-16T09:59:07,795 DEBUG [RS:0;037c58a19ddb:37267-shortCompactions-0 {}] compactions.Compactor(225): Compacting c828236206584489b0b4b191772c331b, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=159, earliestPutTs=1731751146251 2024-11-16T09:59:07,803 INFO [RS:0;037c58a19ddb:37267-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): f3ed23ac403772e0fe4ca9ee892544ec#C#compaction#242 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-16T09:59:07,804 DEBUG [RS:0;037c58a19ddb:37267-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/f3ed23ac403772e0fe4ca9ee892544ec/.tmp/C/98ad7c5b73d9470bbe63c231e0379a5a is 50, key is test_row_0/C:col10/1731751146885/Put/seqid=0 2024-11-16T09:59:07,822 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37591 is added to blk_1073742111_1287 (size=12493) 2024-11-16T09:59:07,830 DEBUG [RS:0;037c58a19ddb:37267-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/f3ed23ac403772e0fe4ca9ee892544ec/.tmp/C/98ad7c5b73d9470bbe63c231e0379a5a as hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/f3ed23ac403772e0fe4ca9ee892544ec/C/98ad7c5b73d9470bbe63c231e0379a5a 2024-11-16T09:59:07,837 INFO [RS:0;037c58a19ddb:37267-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in f3ed23ac403772e0fe4ca9ee892544ec/C of f3ed23ac403772e0fe4ca9ee892544ec into 98ad7c5b73d9470bbe63c231e0379a5a(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-16T09:59:07,837 DEBUG [RS:0;037c58a19ddb:37267-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for f3ed23ac403772e0fe4ca9ee892544ec: 2024-11-16T09:59:07,837 INFO [RS:0;037c58a19ddb:37267-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731751137553.f3ed23ac403772e0fe4ca9ee892544ec., storeName=f3ed23ac403772e0fe4ca9ee892544ec/C, priority=13, startTime=1731751147749; duration=0sec 2024-11-16T09:59:07,837 DEBUG [RS:0;037c58a19ddb:37267-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-16T09:59:07,837 DEBUG [RS:0;037c58a19ddb:37267-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: f3ed23ac403772e0fe4ca9ee892544ec:C 2024-11-16T09:59:08,037 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] regionserver.HRegion(8855): Flush requested on f3ed23ac403772e0fe4ca9ee892544ec 2024-11-16T09:59:08,037 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-16T09:59:08,037 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing f3ed23ac403772e0fe4ca9ee892544ec 3/3 column families, dataSize=60.38 KB heapSize=158.95 KB 2024-11-16T09:59:08,042 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/f3ed23ac403772e0fe4ca9ee892544ec/.tmp/A/726ccf100eb646049bd2e286da649ab2 is 50, key is test_row_0/A:col10/1731751148034/Put/seqid=0 2024-11-16T09:59:08,053 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37591 is added to blk_1073742112_1288 (size=12151) 2024-11-16T09:59:08,053 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=173 (bloomFilter=true), to=hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/f3ed23ac403772e0fe4ca9ee892544ec/.tmp/A/726ccf100eb646049bd2e286da649ab2 2024-11-16T09:59:08,061 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/f3ed23ac403772e0fe4ca9ee892544ec/.tmp/B/192870787ef94f548016d8968e2641c5 is 50, key is test_row_0/B:col10/1731751148034/Put/seqid=0 2024-11-16T09:59:08,066 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37591 is added to blk_1073742113_1289 (size=12151) 2024-11-16T09:59:08,091 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3ed23ac403772e0fe4ca9ee892544ec, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T09:59:08,091 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34368 deadline: 1731751208060, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3ed23ac403772e0fe4ca9ee892544ec, server=037c58a19ddb,37267,1731751077844 2024-11-16T09:59:08,091 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731751137553.f3ed23ac403772e0fe4ca9ee892544ec., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 , the old value is region=TestAcidGuarantees,,1731751137553.f3ed23ac403772e0fe4ca9ee892544ec., hostname=037c58a19ddb,37267,1731751077844, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3ed23ac403772e0fe4ca9ee892544ec, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:59:08,091 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3ed23ac403772e0fe4ca9ee892544ec, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T09:59:08,091 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731751137553.f3ed23ac403772e0fe4ca9ee892544ec., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3ed23ac403772e0fe4ca9ee892544ec, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:59:08,091 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731751137553.f3ed23ac403772e0fe4ca9ee892544ec., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 because the exception is null or not the one we care about 2024-11-16T09:59:08,091 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34414 deadline: 1731751208060, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3ed23ac403772e0fe4ca9ee892544ec, server=037c58a19ddb,37267,1731751077844 2024-11-16T09:59:08,092 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731751137553.f3ed23ac403772e0fe4ca9ee892544ec., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 , the old value is region=TestAcidGuarantees,,1731751137553.f3ed23ac403772e0fe4ca9ee892544ec., hostname=037c58a19ddb,37267,1731751077844, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3ed23ac403772e0fe4ca9ee892544ec, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:59:08,092 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731751137553.f3ed23ac403772e0fe4ca9ee892544ec., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3ed23ac403772e0fe4ca9ee892544ec, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:59:08,092 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731751137553.f3ed23ac403772e0fe4ca9ee892544ec., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 because the exception is null or not the one we care about 2024-11-16T09:59:08,092 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3ed23ac403772e0fe4ca9ee892544ec, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T09:59:08,092 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34382 deadline: 1731751208088, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3ed23ac403772e0fe4ca9ee892544ec, server=037c58a19ddb,37267,1731751077844 2024-11-16T09:59:08,092 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731751137553.f3ed23ac403772e0fe4ca9ee892544ec., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 , the old value is region=TestAcidGuarantees,,1731751137553.f3ed23ac403772e0fe4ca9ee892544ec., hostname=037c58a19ddb,37267,1731751077844, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3ed23ac403772e0fe4ca9ee892544ec, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:59:08,092 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3ed23ac403772e0fe4ca9ee892544ec, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T09:59:08,092 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731751137553.f3ed23ac403772e0fe4ca9ee892544ec., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3ed23ac403772e0fe4ca9ee892544ec, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:59:08,092 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731751137553.f3ed23ac403772e0fe4ca9ee892544ec., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 because the exception is null or not the one we care about 2024-11-16T09:59:08,092 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34408 deadline: 1731751208089, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3ed23ac403772e0fe4ca9ee892544ec, server=037c58a19ddb,37267,1731751077844 2024-11-16T09:59:08,092 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731751137553.f3ed23ac403772e0fe4ca9ee892544ec., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 , the old value is region=TestAcidGuarantees,,1731751137553.f3ed23ac403772e0fe4ca9ee892544ec., hostname=037c58a19ddb,37267,1731751077844, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3ed23ac403772e0fe4ca9ee892544ec, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:59:08,093 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731751137553.f3ed23ac403772e0fe4ca9ee892544ec., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3ed23ac403772e0fe4ca9ee892544ec, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:59:08,093 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731751137553.f3ed23ac403772e0fe4ca9ee892544ec., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 because the exception is null or not the one we care about 2024-11-16T09:59:08,092 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3ed23ac403772e0fe4ca9ee892544ec, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T09:59:08,093 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34386 deadline: 1731751208089, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3ed23ac403772e0fe4ca9ee892544ec, server=037c58a19ddb,37267,1731751077844 2024-11-16T09:59:08,093 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731751137553.f3ed23ac403772e0fe4ca9ee892544ec., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 , the old value is region=TestAcidGuarantees,,1731751137553.f3ed23ac403772e0fe4ca9ee892544ec., hostname=037c58a19ddb,37267,1731751077844, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3ed23ac403772e0fe4ca9ee892544ec, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:59:08,093 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731751137553.f3ed23ac403772e0fe4ca9ee892544ec., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3ed23ac403772e0fe4ca9ee892544ec, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:59:08,093 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731751137553.f3ed23ac403772e0fe4ca9ee892544ec., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 because the exception is null or not the one we care about 2024-11-16T09:59:08,185 DEBUG [RS:0;037c58a19ddb:37267-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/f3ed23ac403772e0fe4ca9ee892544ec/.tmp/B/59e77647903f4960ababcd7f53bcaae7 as hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/f3ed23ac403772e0fe4ca9ee892544ec/B/59e77647903f4960ababcd7f53bcaae7 2024-11-16T09:59:08,192 INFO [RS:0;037c58a19ddb:37267-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in f3ed23ac403772e0fe4ca9ee892544ec/B of f3ed23ac403772e0fe4ca9ee892544ec into 59e77647903f4960ababcd7f53bcaae7(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-16T09:59:08,192 DEBUG [RS:0;037c58a19ddb:37267-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for f3ed23ac403772e0fe4ca9ee892544ec: 2024-11-16T09:59:08,192 INFO [RS:0;037c58a19ddb:37267-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731751137553.f3ed23ac403772e0fe4ca9ee892544ec., storeName=f3ed23ac403772e0fe4ca9ee892544ec/B, priority=13, startTime=1731751147749; duration=0sec 2024-11-16T09:59:08,192 DEBUG [RS:0;037c58a19ddb:37267-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-16T09:59:08,192 DEBUG [RS:0;037c58a19ddb:37267-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: f3ed23ac403772e0fe4ca9ee892544ec:B 2024-11-16T09:59:08,205 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3ed23ac403772e0fe4ca9ee892544ec, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T09:59:08,205 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34368 deadline: 1731751208203, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3ed23ac403772e0fe4ca9ee892544ec, server=037c58a19ddb,37267,1731751077844 2024-11-16T09:59:08,205 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731751137553.f3ed23ac403772e0fe4ca9ee892544ec., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 , the old value is region=TestAcidGuarantees,,1731751137553.f3ed23ac403772e0fe4ca9ee892544ec., hostname=037c58a19ddb,37267,1731751077844, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3ed23ac403772e0fe4ca9ee892544ec, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:59:08,205 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3ed23ac403772e0fe4ca9ee892544ec, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T09:59:08,205 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731751137553.f3ed23ac403772e0fe4ca9ee892544ec., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3ed23ac403772e0fe4ca9ee892544ec, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:59:08,205 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731751137553.f3ed23ac403772e0fe4ca9ee892544ec., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 because the exception is null or not the one we care about 2024-11-16T09:59:08,205 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34414 deadline: 1731751208203, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3ed23ac403772e0fe4ca9ee892544ec, server=037c58a19ddb,37267,1731751077844 2024-11-16T09:59:08,206 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731751137553.f3ed23ac403772e0fe4ca9ee892544ec., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 , the old value is region=TestAcidGuarantees,,1731751137553.f3ed23ac403772e0fe4ca9ee892544ec., hostname=037c58a19ddb,37267,1731751077844, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3ed23ac403772e0fe4ca9ee892544ec, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:59:08,206 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3ed23ac403772e0fe4ca9ee892544ec, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T09:59:08,206 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731751137553.f3ed23ac403772e0fe4ca9ee892544ec., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3ed23ac403772e0fe4ca9ee892544ec, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:59:08,206 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731751137553.f3ed23ac403772e0fe4ca9ee892544ec., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 because the exception is null or not the one we care about 2024-11-16T09:59:08,206 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34382 deadline: 1731751208203, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3ed23ac403772e0fe4ca9ee892544ec, server=037c58a19ddb,37267,1731751077844 2024-11-16T09:59:08,206 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3ed23ac403772e0fe4ca9ee892544ec, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T09:59:08,206 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731751137553.f3ed23ac403772e0fe4ca9ee892544ec., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 , the old value is region=TestAcidGuarantees,,1731751137553.f3ed23ac403772e0fe4ca9ee892544ec., hostname=037c58a19ddb,37267,1731751077844, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3ed23ac403772e0fe4ca9ee892544ec, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:59:08,206 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731751137553.f3ed23ac403772e0fe4ca9ee892544ec., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3ed23ac403772e0fe4ca9ee892544ec, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:59:08,206 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34408 deadline: 1731751208203, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3ed23ac403772e0fe4ca9ee892544ec, server=037c58a19ddb,37267,1731751077844 2024-11-16T09:59:08,206 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731751137553.f3ed23ac403772e0fe4ca9ee892544ec., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 because the exception is null or not the one we care about 2024-11-16T09:59:08,206 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731751137553.f3ed23ac403772e0fe4ca9ee892544ec., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 , the old value is region=TestAcidGuarantees,,1731751137553.f3ed23ac403772e0fe4ca9ee892544ec., hostname=037c58a19ddb,37267,1731751077844, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3ed23ac403772e0fe4ca9ee892544ec, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:59:08,206 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3ed23ac403772e0fe4ca9ee892544ec, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T09:59:08,206 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731751137553.f3ed23ac403772e0fe4ca9ee892544ec., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3ed23ac403772e0fe4ca9ee892544ec, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:59:08,206 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731751137553.f3ed23ac403772e0fe4ca9ee892544ec., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 because the exception is null or not the one we care about 2024-11-16T09:59:08,206 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34386 deadline: 1731751208203, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3ed23ac403772e0fe4ca9ee892544ec, server=037c58a19ddb,37267,1731751077844 2024-11-16T09:59:08,207 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731751137553.f3ed23ac403772e0fe4ca9ee892544ec., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 , the old value is region=TestAcidGuarantees,,1731751137553.f3ed23ac403772e0fe4ca9ee892544ec., hostname=037c58a19ddb,37267,1731751077844, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3ed23ac403772e0fe4ca9ee892544ec, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:59:08,207 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731751137553.f3ed23ac403772e0fe4ca9ee892544ec., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3ed23ac403772e0fe4ca9ee892544ec, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:59:08,207 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731751137553.f3ed23ac403772e0fe4ca9ee892544ec., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 because the exception is null or not the one we care about 2024-11-16T09:59:08,343 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37877 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=66 2024-11-16T09:59:08,343 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-16T09:59:08,343 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37877 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-16T09:59:08,344 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37877 {}] procedure2.ProcedureExecutor(1139): Stored pid=68, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=68, table=TestAcidGuarantees 2024-11-16T09:59:08,345 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37877 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=68 2024-11-16T09:59:08,345 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=68, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=68, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-16T09:59:08,346 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=68, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=68, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-16T09:59:08,346 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=69, ppid=68, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-16T09:59:08,415 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3ed23ac403772e0fe4ca9ee892544ec, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T09:59:08,415 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34368 deadline: 1731751208413, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3ed23ac403772e0fe4ca9ee892544ec, server=037c58a19ddb,37267,1731751077844 2024-11-16T09:59:08,415 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731751137553.f3ed23ac403772e0fe4ca9ee892544ec., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 , the old value is region=TestAcidGuarantees,,1731751137553.f3ed23ac403772e0fe4ca9ee892544ec., hostname=037c58a19ddb,37267,1731751077844, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3ed23ac403772e0fe4ca9ee892544ec, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:59:08,415 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731751137553.f3ed23ac403772e0fe4ca9ee892544ec., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3ed23ac403772e0fe4ca9ee892544ec, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:59:08,415 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731751137553.f3ed23ac403772e0fe4ca9ee892544ec., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 because the exception is null or not the one we care about 2024-11-16T09:59:08,416 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3ed23ac403772e0fe4ca9ee892544ec, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T09:59:08,416 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34414 deadline: 1731751208413, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3ed23ac403772e0fe4ca9ee892544ec, server=037c58a19ddb,37267,1731751077844 2024-11-16T09:59:08,416 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3ed23ac403772e0fe4ca9ee892544ec, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T09:59:08,417 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3ed23ac403772e0fe4ca9ee892544ec, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T09:59:08,417 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34382 deadline: 1731751208413, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3ed23ac403772e0fe4ca9ee892544ec, server=037c58a19ddb,37267,1731751077844 2024-11-16T09:59:08,417 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731751137553.f3ed23ac403772e0fe4ca9ee892544ec., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 , the old value is region=TestAcidGuarantees,,1731751137553.f3ed23ac403772e0fe4ca9ee892544ec., hostname=037c58a19ddb,37267,1731751077844, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3ed23ac403772e0fe4ca9ee892544ec, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:59:08,417 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731751137553.f3ed23ac403772e0fe4ca9ee892544ec., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3ed23ac403772e0fe4ca9ee892544ec, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:59:08,417 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34408 deadline: 1731751208414, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3ed23ac403772e0fe4ca9ee892544ec, server=037c58a19ddb,37267,1731751077844 2024-11-16T09:59:08,417 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731751137553.f3ed23ac403772e0fe4ca9ee892544ec., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 because the exception is null or not the one we care about 2024-11-16T09:59:08,417 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3ed23ac403772e0fe4ca9ee892544ec, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T09:59:08,417 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731751137553.f3ed23ac403772e0fe4ca9ee892544ec., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 , the old value is region=TestAcidGuarantees,,1731751137553.f3ed23ac403772e0fe4ca9ee892544ec., hostname=037c58a19ddb,37267,1731751077844, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3ed23ac403772e0fe4ca9ee892544ec, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:59:08,417 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34386 deadline: 1731751208414, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3ed23ac403772e0fe4ca9ee892544ec, server=037c58a19ddb,37267,1731751077844 2024-11-16T09:59:08,417 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731751137553.f3ed23ac403772e0fe4ca9ee892544ec., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3ed23ac403772e0fe4ca9ee892544ec, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:59:08,417 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731751137553.f3ed23ac403772e0fe4ca9ee892544ec., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 , the old value is region=TestAcidGuarantees,,1731751137553.f3ed23ac403772e0fe4ca9ee892544ec., hostname=037c58a19ddb,37267,1731751077844, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3ed23ac403772e0fe4ca9ee892544ec, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:59:08,417 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731751137553.f3ed23ac403772e0fe4ca9ee892544ec., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 because the exception is null or not the one we care about 2024-11-16T09:59:08,417 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731751137553.f3ed23ac403772e0fe4ca9ee892544ec., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3ed23ac403772e0fe4ca9ee892544ec, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:59:08,417 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731751137553.f3ed23ac403772e0fe4ca9ee892544ec., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 because the exception is null or not the one we care about 2024-11-16T09:59:08,417 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731751137553.f3ed23ac403772e0fe4ca9ee892544ec., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 , the old value is region=TestAcidGuarantees,,1731751137553.f3ed23ac403772e0fe4ca9ee892544ec., hostname=037c58a19ddb,37267,1731751077844, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3ed23ac403772e0fe4ca9ee892544ec, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:59:08,417 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731751137553.f3ed23ac403772e0fe4ca9ee892544ec., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3ed23ac403772e0fe4ca9ee892544ec, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:59:08,417 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731751137553.f3ed23ac403772e0fe4ca9ee892544ec., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 because the exception is null or not the one we care about 2024-11-16T09:59:08,453 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37877 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=68 2024-11-16T09:59:08,467 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=173 (bloomFilter=true), to=hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/f3ed23ac403772e0fe4ca9ee892544ec/.tmp/B/192870787ef94f548016d8968e2641c5 2024-11-16T09:59:08,474 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/f3ed23ac403772e0fe4ca9ee892544ec/.tmp/C/515f53734f6e4430b06393764cb56c0f is 50, key is test_row_0/C:col10/1731751148034/Put/seqid=0 2024-11-16T09:59:08,478 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37591 is added to blk_1073742114_1290 (size=12151) 2024-11-16T09:59:08,478 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=173 (bloomFilter=true), to=hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/f3ed23ac403772e0fe4ca9ee892544ec/.tmp/C/515f53734f6e4430b06393764cb56c0f 2024-11-16T09:59:08,482 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/f3ed23ac403772e0fe4ca9ee892544ec/.tmp/A/726ccf100eb646049bd2e286da649ab2 as hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/f3ed23ac403772e0fe4ca9ee892544ec/A/726ccf100eb646049bd2e286da649ab2 2024-11-16T09:59:08,487 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/f3ed23ac403772e0fe4ca9ee892544ec/A/726ccf100eb646049bd2e286da649ab2, entries=150, sequenceid=173, filesize=11.9 K 2024-11-16T09:59:08,488 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/f3ed23ac403772e0fe4ca9ee892544ec/.tmp/B/192870787ef94f548016d8968e2641c5 as hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/f3ed23ac403772e0fe4ca9ee892544ec/B/192870787ef94f548016d8968e2641c5 2024-11-16T09:59:08,493 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/f3ed23ac403772e0fe4ca9ee892544ec/B/192870787ef94f548016d8968e2641c5, entries=150, sequenceid=173, filesize=11.9 K 2024-11-16T09:59:08,494 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/f3ed23ac403772e0fe4ca9ee892544ec/.tmp/C/515f53734f6e4430b06393764cb56c0f as hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/f3ed23ac403772e0fe4ca9ee892544ec/C/515f53734f6e4430b06393764cb56c0f 2024-11-16T09:59:08,499 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=37267 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=69 2024-11-16T09:59:08,499 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/f3ed23ac403772e0fe4ca9ee892544ec/C/515f53734f6e4430b06393764cb56c0f, entries=150, sequenceid=173, filesize=11.9 K 2024-11-16T09:59:08,499 DEBUG [RS_FLUSH_OPERATIONS-regionserver/037c58a19ddb:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731751137553.f3ed23ac403772e0fe4ca9ee892544ec. 2024-11-16T09:59:08,499 DEBUG [RS_FLUSH_OPERATIONS-regionserver/037c58a19ddb:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731751137553.f3ed23ac403772e0fe4ca9ee892544ec. as already flushing 2024-11-16T09:59:08,499 DEBUG [RS_FLUSH_OPERATIONS-regionserver/037c58a19ddb:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731751137553.f3ed23ac403772e0fe4ca9ee892544ec. 2024-11-16T09:59:08,499 ERROR [RS_FLUSH_OPERATIONS-regionserver/037c58a19ddb:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] handler.RSProcedureHandler(58): pid=69 java.io.IOException: Unable to complete flush {ENCODED => f3ed23ac403772e0fe4ca9ee892544ec, NAME => 'TestAcidGuarantees,,1731751137553.f3ed23ac403772e0fe4ca9ee892544ec.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T09:59:08,499 DEBUG [RS_FLUSH_OPERATIONS-regionserver/037c58a19ddb:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=69 java.io.IOException: Unable to complete flush {ENCODED => f3ed23ac403772e0fe4ca9ee892544ec, NAME => 'TestAcidGuarantees,,1731751137553.f3ed23ac403772e0fe4ca9ee892544ec.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T09:59:08,500 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~158.91 KB/162720, currentSize=140.89 KB/144270 for f3ed23ac403772e0fe4ca9ee892544ec in 463ms, sequenceid=173, compaction requested=false 2024-11-16T09:59:08,500 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for f3ed23ac403772e0fe4ca9ee892544ec: 2024-11-16T09:59:08,500 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37877 {}] master.HMaster(4177): Remote procedure failed, pid=69 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => f3ed23ac403772e0fe4ca9ee892544ec, NAME => 'TestAcidGuarantees,,1731751137553.f3ed23ac403772e0fe4ca9ee892544ec.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => f3ed23ac403772e0fe4ca9ee892544ec, NAME => 'TestAcidGuarantees,,1731751137553.f3ed23ac403772e0fe4ca9ee892544ec.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T09:59:08,651 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=37267 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=69 2024-11-16T09:59:08,652 DEBUG [RS_FLUSH_OPERATIONS-regionserver/037c58a19ddb:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731751137553.f3ed23ac403772e0fe4ca9ee892544ec. 2024-11-16T09:59:08,652 INFO [RS_FLUSH_OPERATIONS-regionserver/037c58a19ddb:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HRegion(2902): Flushing f3ed23ac403772e0fe4ca9ee892544ec 3/3 column families, dataSize=140.89 KB heapSize=369.89 KB 2024-11-16T09:59:08,657 DEBUG [RS_FLUSH_OPERATIONS-regionserver/037c58a19ddb:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/f3ed23ac403772e0fe4ca9ee892544ec/.tmp/A/2a41a1f803ea4e6e9929313e235b0236 is 50, key is test_row_0/A:col10/1731751148062/Put/seqid=0 2024-11-16T09:59:08,662 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37591 is added to blk_1073742115_1291 (size=12151) 2024-11-16T09:59:08,662 INFO [RS_FLUSH_OPERATIONS-regionserver/037c58a19ddb:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=198 (bloomFilter=true), to=hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/f3ed23ac403772e0fe4ca9ee892544ec/.tmp/A/2a41a1f803ea4e6e9929313e235b0236 2024-11-16T09:59:08,663 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37877 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=68 2024-11-16T09:59:08,671 DEBUG [RS_FLUSH_OPERATIONS-regionserver/037c58a19ddb:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/f3ed23ac403772e0fe4ca9ee892544ec/.tmp/B/97437b43d51e496ba02e0a3029c174a6 is 50, key is test_row_0/B:col10/1731751148062/Put/seqid=0 2024-11-16T09:59:08,675 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37591 is added to blk_1073742116_1292 (size=12151) 2024-11-16T09:59:08,725 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] regionserver.HRegion(8855): Flush requested on f3ed23ac403772e0fe4ca9ee892544ec 2024-11-16T09:59:08,725 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731751137553.f3ed23ac403772e0fe4ca9ee892544ec. as already flushing 2024-11-16T09:59:08,733 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3ed23ac403772e0fe4ca9ee892544ec, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T09:59:08,734 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34368 deadline: 1731751208730, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3ed23ac403772e0fe4ca9ee892544ec, server=037c58a19ddb,37267,1731751077844 2024-11-16T09:59:08,734 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731751137553.f3ed23ac403772e0fe4ca9ee892544ec., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 , the old value is region=TestAcidGuarantees,,1731751137553.f3ed23ac403772e0fe4ca9ee892544ec., hostname=037c58a19ddb,37267,1731751077844, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3ed23ac403772e0fe4ca9ee892544ec, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:59:08,734 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731751137553.f3ed23ac403772e0fe4ca9ee892544ec., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3ed23ac403772e0fe4ca9ee892544ec, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:59:08,734 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731751137553.f3ed23ac403772e0fe4ca9ee892544ec., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 because the exception is null or not the one we care about 2024-11-16T09:59:08,737 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3ed23ac403772e0fe4ca9ee892544ec, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T09:59:08,737 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3ed23ac403772e0fe4ca9ee892544ec, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T09:59:08,737 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34408 deadline: 1731751208732, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3ed23ac403772e0fe4ca9ee892544ec, server=037c58a19ddb,37267,1731751077844 2024-11-16T09:59:08,737 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34414 deadline: 1731751208732, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3ed23ac403772e0fe4ca9ee892544ec, server=037c58a19ddb,37267,1731751077844 2024-11-16T09:59:08,737 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3ed23ac403772e0fe4ca9ee892544ec, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T09:59:08,737 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3ed23ac403772e0fe4ca9ee892544ec, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T09:59:08,737 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731751137553.f3ed23ac403772e0fe4ca9ee892544ec., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 , the old value is region=TestAcidGuarantees,,1731751137553.f3ed23ac403772e0fe4ca9ee892544ec., hostname=037c58a19ddb,37267,1731751077844, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3ed23ac403772e0fe4ca9ee892544ec, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:59:08,737 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731751137553.f3ed23ac403772e0fe4ca9ee892544ec., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3ed23ac403772e0fe4ca9ee892544ec, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:59:08,737 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34386 deadline: 1731751208733, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3ed23ac403772e0fe4ca9ee892544ec, server=037c58a19ddb,37267,1731751077844 2024-11-16T09:59:08,737 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34382 deadline: 1731751208733, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3ed23ac403772e0fe4ca9ee892544ec, server=037c58a19ddb,37267,1731751077844 2024-11-16T09:59:08,737 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731751137553.f3ed23ac403772e0fe4ca9ee892544ec., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 because the exception is null or not the one we care about 2024-11-16T09:59:08,738 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731751137553.f3ed23ac403772e0fe4ca9ee892544ec., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 , the old value is region=TestAcidGuarantees,,1731751137553.f3ed23ac403772e0fe4ca9ee892544ec., hostname=037c58a19ddb,37267,1731751077844, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3ed23ac403772e0fe4ca9ee892544ec, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:59:08,738 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731751137553.f3ed23ac403772e0fe4ca9ee892544ec., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3ed23ac403772e0fe4ca9ee892544ec, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:59:08,738 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731751137553.f3ed23ac403772e0fe4ca9ee892544ec., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 because the exception is null or not the one we care about 2024-11-16T09:59:08,738 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731751137553.f3ed23ac403772e0fe4ca9ee892544ec., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 , the old value is region=TestAcidGuarantees,,1731751137553.f3ed23ac403772e0fe4ca9ee892544ec., hostname=037c58a19ddb,37267,1731751077844, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3ed23ac403772e0fe4ca9ee892544ec, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:59:08,738 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731751137553.f3ed23ac403772e0fe4ca9ee892544ec., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3ed23ac403772e0fe4ca9ee892544ec, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:59:08,738 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731751137553.f3ed23ac403772e0fe4ca9ee892544ec., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 because the exception is null or not the one we care about 2024-11-16T09:59:08,738 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731751137553.f3ed23ac403772e0fe4ca9ee892544ec., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 , the old value is region=TestAcidGuarantees,,1731751137553.f3ed23ac403772e0fe4ca9ee892544ec., hostname=037c58a19ddb,37267,1731751077844, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3ed23ac403772e0fe4ca9ee892544ec, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:59:08,738 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731751137553.f3ed23ac403772e0fe4ca9ee892544ec., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3ed23ac403772e0fe4ca9ee892544ec, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:59:08,738 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731751137553.f3ed23ac403772e0fe4ca9ee892544ec., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 because the exception is null or not the one we care about 2024-11-16T09:59:08,845 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3ed23ac403772e0fe4ca9ee892544ec, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T09:59:08,845 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3ed23ac403772e0fe4ca9ee892544ec, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T09:59:08,845 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34368 deadline: 1731751208843, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3ed23ac403772e0fe4ca9ee892544ec, server=037c58a19ddb,37267,1731751077844 2024-11-16T09:59:08,845 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34408 deadline: 1731751208843, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3ed23ac403772e0fe4ca9ee892544ec, server=037c58a19ddb,37267,1731751077844 2024-11-16T09:59:08,845 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3ed23ac403772e0fe4ca9ee892544ec, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T09:59:08,845 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731751137553.f3ed23ac403772e0fe4ca9ee892544ec., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 , the old value is region=TestAcidGuarantees,,1731751137553.f3ed23ac403772e0fe4ca9ee892544ec., hostname=037c58a19ddb,37267,1731751077844, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3ed23ac403772e0fe4ca9ee892544ec, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:59:08,845 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731751137553.f3ed23ac403772e0fe4ca9ee892544ec., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3ed23ac403772e0fe4ca9ee892544ec, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:59:08,845 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731751137553.f3ed23ac403772e0fe4ca9ee892544ec., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 because the exception is null or not the one we care about 2024-11-16T09:59:08,845 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34414 deadline: 1731751208843, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3ed23ac403772e0fe4ca9ee892544ec, server=037c58a19ddb,37267,1731751077844 2024-11-16T09:59:08,846 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3ed23ac403772e0fe4ca9ee892544ec, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T09:59:08,846 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731751137553.f3ed23ac403772e0fe4ca9ee892544ec., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 , the old value is region=TestAcidGuarantees,,1731751137553.f3ed23ac403772e0fe4ca9ee892544ec., hostname=037c58a19ddb,37267,1731751077844, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3ed23ac403772e0fe4ca9ee892544ec, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:59:08,846 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731751137553.f3ed23ac403772e0fe4ca9ee892544ec., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3ed23ac403772e0fe4ca9ee892544ec, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:59:08,846 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731751137553.f3ed23ac403772e0fe4ca9ee892544ec., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 , the old value is region=TestAcidGuarantees,,1731751137553.f3ed23ac403772e0fe4ca9ee892544ec., hostname=037c58a19ddb,37267,1731751077844, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3ed23ac403772e0fe4ca9ee892544ec, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:59:08,846 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34386 deadline: 1731751208843, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3ed23ac403772e0fe4ca9ee892544ec, server=037c58a19ddb,37267,1731751077844 2024-11-16T09:59:08,846 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731751137553.f3ed23ac403772e0fe4ca9ee892544ec., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 because the exception is null or not the one we care about 2024-11-16T09:59:08,846 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731751137553.f3ed23ac403772e0fe4ca9ee892544ec., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3ed23ac403772e0fe4ca9ee892544ec, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:59:08,846 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731751137553.f3ed23ac403772e0fe4ca9ee892544ec., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 because the exception is null or not the one we care about 2024-11-16T09:59:08,846 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3ed23ac403772e0fe4ca9ee892544ec, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T09:59:08,846 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731751137553.f3ed23ac403772e0fe4ca9ee892544ec., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 , the old value is region=TestAcidGuarantees,,1731751137553.f3ed23ac403772e0fe4ca9ee892544ec., hostname=037c58a19ddb,37267,1731751077844, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3ed23ac403772e0fe4ca9ee892544ec, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:59:08,846 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34382 deadline: 1731751208843, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3ed23ac403772e0fe4ca9ee892544ec, server=037c58a19ddb,37267,1731751077844 2024-11-16T09:59:08,846 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731751137553.f3ed23ac403772e0fe4ca9ee892544ec., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3ed23ac403772e0fe4ca9ee892544ec, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:59:08,846 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731751137553.f3ed23ac403772e0fe4ca9ee892544ec., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 because the exception is null or not the one we care about 2024-11-16T09:59:08,846 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731751137553.f3ed23ac403772e0fe4ca9ee892544ec., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 , the old value is region=TestAcidGuarantees,,1731751137553.f3ed23ac403772e0fe4ca9ee892544ec., hostname=037c58a19ddb,37267,1731751077844, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3ed23ac403772e0fe4ca9ee892544ec, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:59:08,846 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731751137553.f3ed23ac403772e0fe4ca9ee892544ec., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3ed23ac403772e0fe4ca9ee892544ec, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:59:08,846 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731751137553.f3ed23ac403772e0fe4ca9ee892544ec., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 because the exception is null or not the one we care about 2024-11-16T09:59:08,972 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37877 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=68 2024-11-16T09:59:09,055 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3ed23ac403772e0fe4ca9ee892544ec, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T09:59:09,055 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34368 deadline: 1731751209053, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3ed23ac403772e0fe4ca9ee892544ec, server=037c58a19ddb,37267,1731751077844 2024-11-16T09:59:09,055 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3ed23ac403772e0fe4ca9ee892544ec, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T09:59:09,055 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731751137553.f3ed23ac403772e0fe4ca9ee892544ec., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 , the old value is region=TestAcidGuarantees,,1731751137553.f3ed23ac403772e0fe4ca9ee892544ec., hostname=037c58a19ddb,37267,1731751077844, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3ed23ac403772e0fe4ca9ee892544ec, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:59:09,055 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34414 deadline: 1731751209053, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3ed23ac403772e0fe4ca9ee892544ec, server=037c58a19ddb,37267,1731751077844 2024-11-16T09:59:09,055 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731751137553.f3ed23ac403772e0fe4ca9ee892544ec., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3ed23ac403772e0fe4ca9ee892544ec, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:59:09,055 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731751137553.f3ed23ac403772e0fe4ca9ee892544ec., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 because the exception is null or not the one we care about 2024-11-16T09:59:09,055 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3ed23ac403772e0fe4ca9ee892544ec, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T09:59:09,056 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34408 deadline: 1731751209053, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3ed23ac403772e0fe4ca9ee892544ec, server=037c58a19ddb,37267,1731751077844 2024-11-16T09:59:09,056 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731751137553.f3ed23ac403772e0fe4ca9ee892544ec., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 , the old value is region=TestAcidGuarantees,,1731751137553.f3ed23ac403772e0fe4ca9ee892544ec., hostname=037c58a19ddb,37267,1731751077844, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3ed23ac403772e0fe4ca9ee892544ec, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:59:09,056 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3ed23ac403772e0fe4ca9ee892544ec, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T09:59:09,056 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731751137553.f3ed23ac403772e0fe4ca9ee892544ec., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3ed23ac403772e0fe4ca9ee892544ec, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:59:09,056 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731751137553.f3ed23ac403772e0fe4ca9ee892544ec., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 because the exception is null or not the one we care about 2024-11-16T09:59:09,056 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34386 deadline: 1731751209053, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3ed23ac403772e0fe4ca9ee892544ec, server=037c58a19ddb,37267,1731751077844 2024-11-16T09:59:09,056 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731751137553.f3ed23ac403772e0fe4ca9ee892544ec., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 , the old value is region=TestAcidGuarantees,,1731751137553.f3ed23ac403772e0fe4ca9ee892544ec., hostname=037c58a19ddb,37267,1731751077844, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3ed23ac403772e0fe4ca9ee892544ec, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:59:09,056 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3ed23ac403772e0fe4ca9ee892544ec, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T09:59:09,056 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731751137553.f3ed23ac403772e0fe4ca9ee892544ec., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3ed23ac403772e0fe4ca9ee892544ec, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:59:09,056 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731751137553.f3ed23ac403772e0fe4ca9ee892544ec., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 because the exception is null or not the one we care about 2024-11-16T09:59:09,056 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731751137553.f3ed23ac403772e0fe4ca9ee892544ec., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 , the old value is region=TestAcidGuarantees,,1731751137553.f3ed23ac403772e0fe4ca9ee892544ec., hostname=037c58a19ddb,37267,1731751077844, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3ed23ac403772e0fe4ca9ee892544ec, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:59:09,056 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34382 deadline: 1731751209054, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3ed23ac403772e0fe4ca9ee892544ec, server=037c58a19ddb,37267,1731751077844 2024-11-16T09:59:09,056 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731751137553.f3ed23ac403772e0fe4ca9ee892544ec., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3ed23ac403772e0fe4ca9ee892544ec, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:59:09,056 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731751137553.f3ed23ac403772e0fe4ca9ee892544ec., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 because the exception is null or not the one we care about 2024-11-16T09:59:09,056 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731751137553.f3ed23ac403772e0fe4ca9ee892544ec., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 , the old value is region=TestAcidGuarantees,,1731751137553.f3ed23ac403772e0fe4ca9ee892544ec., hostname=037c58a19ddb,37267,1731751077844, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3ed23ac403772e0fe4ca9ee892544ec, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:59:09,056 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731751137553.f3ed23ac403772e0fe4ca9ee892544ec., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3ed23ac403772e0fe4ca9ee892544ec, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:59:09,056 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731751137553.f3ed23ac403772e0fe4ca9ee892544ec., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 because the exception is null or not the one we care about 2024-11-16T09:59:09,075 INFO [RS_FLUSH_OPERATIONS-regionserver/037c58a19ddb:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=198 (bloomFilter=true), to=hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/f3ed23ac403772e0fe4ca9ee892544ec/.tmp/B/97437b43d51e496ba02e0a3029c174a6 2024-11-16T09:59:09,084 DEBUG [RS_FLUSH_OPERATIONS-regionserver/037c58a19ddb:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/f3ed23ac403772e0fe4ca9ee892544ec/.tmp/C/a4edb180540e4bb7a2083c0f6df737f7 is 50, key is test_row_0/C:col10/1731751148062/Put/seqid=0 2024-11-16T09:59:09,087 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37591 is added to blk_1073742117_1293 (size=12151) 2024-11-16T09:59:09,366 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3ed23ac403772e0fe4ca9ee892544ec, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T09:59:09,366 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34368 deadline: 1731751209363, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3ed23ac403772e0fe4ca9ee892544ec, server=037c58a19ddb,37267,1731751077844 2024-11-16T09:59:09,366 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3ed23ac403772e0fe4ca9ee892544ec, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T09:59:09,366 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731751137553.f3ed23ac403772e0fe4ca9ee892544ec., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 , the old value is region=TestAcidGuarantees,,1731751137553.f3ed23ac403772e0fe4ca9ee892544ec., hostname=037c58a19ddb,37267,1731751077844, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3ed23ac403772e0fe4ca9ee892544ec, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:59:09,366 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3ed23ac403772e0fe4ca9ee892544ec, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T09:59:09,366 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731751137553.f3ed23ac403772e0fe4ca9ee892544ec., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3ed23ac403772e0fe4ca9ee892544ec, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:59:09,366 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731751137553.f3ed23ac403772e0fe4ca9ee892544ec., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 because the exception is null or not the one we care about 2024-11-16T09:59:09,366 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34408 deadline: 1731751209363, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3ed23ac403772e0fe4ca9ee892544ec, server=037c58a19ddb,37267,1731751077844 2024-11-16T09:59:09,366 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34414 deadline: 1731751209363, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3ed23ac403772e0fe4ca9ee892544ec, server=037c58a19ddb,37267,1731751077844 2024-11-16T09:59:09,366 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3ed23ac403772e0fe4ca9ee892544ec, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T09:59:09,366 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731751137553.f3ed23ac403772e0fe4ca9ee892544ec., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 , the old value is region=TestAcidGuarantees,,1731751137553.f3ed23ac403772e0fe4ca9ee892544ec., hostname=037c58a19ddb,37267,1731751077844, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3ed23ac403772e0fe4ca9ee892544ec, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:59:09,366 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3ed23ac403772e0fe4ca9ee892544ec, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T09:59:09,367 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] ipc.CallRunner(138): callId: 71 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34386 deadline: 1731751209363, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3ed23ac403772e0fe4ca9ee892544ec, server=037c58a19ddb,37267,1731751077844 2024-11-16T09:59:09,367 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731751137553.f3ed23ac403772e0fe4ca9ee892544ec., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3ed23ac403772e0fe4ca9ee892544ec, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:59:09,367 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731751137553.f3ed23ac403772e0fe4ca9ee892544ec., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 because the exception is null or not the one we care about 2024-11-16T09:59:09,367 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34382 deadline: 1731751209363, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3ed23ac403772e0fe4ca9ee892544ec, server=037c58a19ddb,37267,1731751077844 2024-11-16T09:59:09,367 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731751137553.f3ed23ac403772e0fe4ca9ee892544ec., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 , the old value is region=TestAcidGuarantees,,1731751137553.f3ed23ac403772e0fe4ca9ee892544ec., hostname=037c58a19ddb,37267,1731751077844, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3ed23ac403772e0fe4ca9ee892544ec, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:59:09,367 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731751137553.f3ed23ac403772e0fe4ca9ee892544ec., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3ed23ac403772e0fe4ca9ee892544ec, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:59:09,367 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731751137553.f3ed23ac403772e0fe4ca9ee892544ec., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 because the exception is null or not the one we care about 2024-11-16T09:59:09,367 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731751137553.f3ed23ac403772e0fe4ca9ee892544ec., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 , the old value is region=TestAcidGuarantees,,1731751137553.f3ed23ac403772e0fe4ca9ee892544ec., hostname=037c58a19ddb,37267,1731751077844, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3ed23ac403772e0fe4ca9ee892544ec, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:59:09,367 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731751137553.f3ed23ac403772e0fe4ca9ee892544ec., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 , the old value is region=TestAcidGuarantees,,1731751137553.f3ed23ac403772e0fe4ca9ee892544ec., hostname=037c58a19ddb,37267,1731751077844, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3ed23ac403772e0fe4ca9ee892544ec, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:59:09,367 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731751137553.f3ed23ac403772e0fe4ca9ee892544ec., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3ed23ac403772e0fe4ca9ee892544ec, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:59:09,367 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731751137553.f3ed23ac403772e0fe4ca9ee892544ec., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 because the exception is null or not the one we care about 2024-11-16T09:59:09,367 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731751137553.f3ed23ac403772e0fe4ca9ee892544ec., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3ed23ac403772e0fe4ca9ee892544ec, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:59:09,367 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731751137553.f3ed23ac403772e0fe4ca9ee892544ec., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 because the exception is null or not the one we care about 2024-11-16T09:59:09,482 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37877 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=68 2024-11-16T09:59:09,488 INFO [RS_FLUSH_OPERATIONS-regionserver/037c58a19ddb:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=198 (bloomFilter=true), to=hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/f3ed23ac403772e0fe4ca9ee892544ec/.tmp/C/a4edb180540e4bb7a2083c0f6df737f7 2024-11-16T09:59:09,493 DEBUG [RS_FLUSH_OPERATIONS-regionserver/037c58a19ddb:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/f3ed23ac403772e0fe4ca9ee892544ec/.tmp/A/2a41a1f803ea4e6e9929313e235b0236 as hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/f3ed23ac403772e0fe4ca9ee892544ec/A/2a41a1f803ea4e6e9929313e235b0236 2024-11-16T09:59:09,497 INFO [RS_FLUSH_OPERATIONS-regionserver/037c58a19ddb:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/f3ed23ac403772e0fe4ca9ee892544ec/A/2a41a1f803ea4e6e9929313e235b0236, entries=150, sequenceid=198, filesize=11.9 K 2024-11-16T09:59:09,498 DEBUG [RS_FLUSH_OPERATIONS-regionserver/037c58a19ddb:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/f3ed23ac403772e0fe4ca9ee892544ec/.tmp/B/97437b43d51e496ba02e0a3029c174a6 as hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/f3ed23ac403772e0fe4ca9ee892544ec/B/97437b43d51e496ba02e0a3029c174a6 2024-11-16T09:59:09,502 INFO [RS_FLUSH_OPERATIONS-regionserver/037c58a19ddb:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/f3ed23ac403772e0fe4ca9ee892544ec/B/97437b43d51e496ba02e0a3029c174a6, entries=150, sequenceid=198, filesize=11.9 K 2024-11-16T09:59:09,502 DEBUG [RS_FLUSH_OPERATIONS-regionserver/037c58a19ddb:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/f3ed23ac403772e0fe4ca9ee892544ec/.tmp/C/a4edb180540e4bb7a2083c0f6df737f7 as hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/f3ed23ac403772e0fe4ca9ee892544ec/C/a4edb180540e4bb7a2083c0f6df737f7 2024-11-16T09:59:09,506 INFO [RS_FLUSH_OPERATIONS-regionserver/037c58a19ddb:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/f3ed23ac403772e0fe4ca9ee892544ec/C/a4edb180540e4bb7a2083c0f6df737f7, entries=150, sequenceid=198, filesize=11.9 K 2024-11-16T09:59:09,507 INFO [RS_FLUSH_OPERATIONS-regionserver/037c58a19ddb:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HRegion(3140): Finished flush of dataSize ~140.89 KB/144270, heapSize ~369.84 KB/378720, currentSize=67.09 KB/68700 for f3ed23ac403772e0fe4ca9ee892544ec in 855ms, sequenceid=198, compaction requested=true 2024-11-16T09:59:09,507 DEBUG [RS_FLUSH_OPERATIONS-regionserver/037c58a19ddb:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HRegion(2603): Flush status journal for f3ed23ac403772e0fe4ca9ee892544ec: 2024-11-16T09:59:09,507 DEBUG [RS_FLUSH_OPERATIONS-regionserver/037c58a19ddb:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731751137553.f3ed23ac403772e0fe4ca9ee892544ec. 2024-11-16T09:59:09,507 DEBUG [RS_FLUSH_OPERATIONS-regionserver/037c58a19ddb:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=69 2024-11-16T09:59:09,508 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37877 {}] master.HMaster(4169): Remote procedure done, pid=69 2024-11-16T09:59:09,511 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=69, resume processing ppid=68 2024-11-16T09:59:09,511 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=69, ppid=68, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.1630 sec 2024-11-16T09:59:09,513 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=68, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=68, table=TestAcidGuarantees in 1.1690 sec 2024-11-16T09:59:09,876 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] regionserver.HRegion(8855): Flush requested on f3ed23ac403772e0fe4ca9ee892544ec 2024-11-16T09:59:09,876 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-16T09:59:09,876 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing f3ed23ac403772e0fe4ca9ee892544ec 3/3 column families, dataSize=80.51 KB heapSize=211.69 KB 2024-11-16T09:59:09,882 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/f3ed23ac403772e0fe4ca9ee892544ec/.tmp/A/e1f68d6740d84ab2b22314b6666d769e is 50, key is test_row_0/A:col10/1731751149876/Put/seqid=0 2024-11-16T09:59:09,886 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37591 is added to blk_1073742118_1294 (size=14541) 2024-11-16T09:59:09,896 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3ed23ac403772e0fe4ca9ee892544ec, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T09:59:09,896 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] ipc.CallRunner(138): callId: 71 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34408 deadline: 1731751209892, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3ed23ac403772e0fe4ca9ee892544ec, server=037c58a19ddb,37267,1731751077844 2024-11-16T09:59:09,897 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731751137553.f3ed23ac403772e0fe4ca9ee892544ec., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 , the old value is region=TestAcidGuarantees,,1731751137553.f3ed23ac403772e0fe4ca9ee892544ec., hostname=037c58a19ddb,37267,1731751077844, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3ed23ac403772e0fe4ca9ee892544ec, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:59:09,897 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731751137553.f3ed23ac403772e0fe4ca9ee892544ec., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3ed23ac403772e0fe4ca9ee892544ec, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:59:09,897 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731751137553.f3ed23ac403772e0fe4ca9ee892544ec., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 because the exception is null or not the one we care about 2024-11-16T09:59:09,900 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3ed23ac403772e0fe4ca9ee892544ec, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T09:59:09,900 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] ipc.CallRunner(138): callId: 72 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34368 deadline: 1731751209895, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3ed23ac403772e0fe4ca9ee892544ec, server=037c58a19ddb,37267,1731751077844 2024-11-16T09:59:09,900 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731751137553.f3ed23ac403772e0fe4ca9ee892544ec., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 , the old value is region=TestAcidGuarantees,,1731751137553.f3ed23ac403772e0fe4ca9ee892544ec., hostname=037c58a19ddb,37267,1731751077844, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3ed23ac403772e0fe4ca9ee892544ec, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:59:09,900 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731751137553.f3ed23ac403772e0fe4ca9ee892544ec., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3ed23ac403772e0fe4ca9ee892544ec, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:59:09,900 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731751137553.f3ed23ac403772e0fe4ca9ee892544ec., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 because the exception is null or not the one we care about 2024-11-16T09:59:09,901 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3ed23ac403772e0fe4ca9ee892544ec, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T09:59:09,901 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] ipc.CallRunner(138): callId: 76 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34386 deadline: 1731751209895, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3ed23ac403772e0fe4ca9ee892544ec, server=037c58a19ddb,37267,1731751077844 2024-11-16T09:59:09,901 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731751137553.f3ed23ac403772e0fe4ca9ee892544ec., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 , the old value is region=TestAcidGuarantees,,1731751137553.f3ed23ac403772e0fe4ca9ee892544ec., hostname=037c58a19ddb,37267,1731751077844, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3ed23ac403772e0fe4ca9ee892544ec, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:59:09,901 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3ed23ac403772e0fe4ca9ee892544ec, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T09:59:09,901 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731751137553.f3ed23ac403772e0fe4ca9ee892544ec., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3ed23ac403772e0fe4ca9ee892544ec, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:59:09,902 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731751137553.f3ed23ac403772e0fe4ca9ee892544ec., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 because the exception is null or not the one we care about 2024-11-16T09:59:09,902 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] ipc.CallRunner(138): callId: 72 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34382 deadline: 1731751209896, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3ed23ac403772e0fe4ca9ee892544ec, server=037c58a19ddb,37267,1731751077844 2024-11-16T09:59:09,902 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731751137553.f3ed23ac403772e0fe4ca9ee892544ec., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 , the old value is region=TestAcidGuarantees,,1731751137553.f3ed23ac403772e0fe4ca9ee892544ec., hostname=037c58a19ddb,37267,1731751077844, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3ed23ac403772e0fe4ca9ee892544ec, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:59:09,902 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731751137553.f3ed23ac403772e0fe4ca9ee892544ec., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3ed23ac403772e0fe4ca9ee892544ec, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:59:09,902 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731751137553.f3ed23ac403772e0fe4ca9ee892544ec., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 because the exception is null or not the one we care about 2024-11-16T09:59:09,902 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3ed23ac403772e0fe4ca9ee892544ec, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T09:59:09,902 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] ipc.CallRunner(138): callId: 72 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34414 deadline: 1731751209897, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3ed23ac403772e0fe4ca9ee892544ec, server=037c58a19ddb,37267,1731751077844 2024-11-16T09:59:09,902 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731751137553.f3ed23ac403772e0fe4ca9ee892544ec., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 , the old value is region=TestAcidGuarantees,,1731751137553.f3ed23ac403772e0fe4ca9ee892544ec., hostname=037c58a19ddb,37267,1731751077844, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3ed23ac403772e0fe4ca9ee892544ec, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:59:09,902 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731751137553.f3ed23ac403772e0fe4ca9ee892544ec., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3ed23ac403772e0fe4ca9ee892544ec, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:59:09,902 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731751137553.f3ed23ac403772e0fe4ca9ee892544ec., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 because the exception is null or not the one we care about 2024-11-16T09:59:10,005 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3ed23ac403772e0fe4ca9ee892544ec, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T09:59:10,006 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] ipc.CallRunner(138): callId: 72 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34408 deadline: 1731751210003, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3ed23ac403772e0fe4ca9ee892544ec, server=037c58a19ddb,37267,1731751077844 2024-11-16T09:59:10,006 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3ed23ac403772e0fe4ca9ee892544ec, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T09:59:10,006 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731751137553.f3ed23ac403772e0fe4ca9ee892544ec., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 , the old value is region=TestAcidGuarantees,,1731751137553.f3ed23ac403772e0fe4ca9ee892544ec., hostname=037c58a19ddb,37267,1731751077844, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3ed23ac403772e0fe4ca9ee892544ec, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:59:10,006 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731751137553.f3ed23ac403772e0fe4ca9ee892544ec., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3ed23ac403772e0fe4ca9ee892544ec, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:59:10,006 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] ipc.CallRunner(138): callId: 73 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34368 deadline: 1731751210003, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3ed23ac403772e0fe4ca9ee892544ec, server=037c58a19ddb,37267,1731751077844 2024-11-16T09:59:10,006 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731751137553.f3ed23ac403772e0fe4ca9ee892544ec., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 because the exception is null or not the one we care about 2024-11-16T09:59:10,006 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731751137553.f3ed23ac403772e0fe4ca9ee892544ec., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 , the old value is region=TestAcidGuarantees,,1731751137553.f3ed23ac403772e0fe4ca9ee892544ec., hostname=037c58a19ddb,37267,1731751077844, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3ed23ac403772e0fe4ca9ee892544ec, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:59:10,006 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731751137553.f3ed23ac403772e0fe4ca9ee892544ec., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3ed23ac403772e0fe4ca9ee892544ec, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:59:10,006 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731751137553.f3ed23ac403772e0fe4ca9ee892544ec., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 because the exception is null or not the one we care about 2024-11-16T09:59:10,015 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3ed23ac403772e0fe4ca9ee892544ec, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T09:59:10,016 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] ipc.CallRunner(138): callId: 77 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34386 deadline: 1731751210013, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3ed23ac403772e0fe4ca9ee892544ec, server=037c58a19ddb,37267,1731751077844 2024-11-16T09:59:10,016 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731751137553.f3ed23ac403772e0fe4ca9ee892544ec., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 , the old value is region=TestAcidGuarantees,,1731751137553.f3ed23ac403772e0fe4ca9ee892544ec., hostname=037c58a19ddb,37267,1731751077844, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3ed23ac403772e0fe4ca9ee892544ec, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:59:10,016 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731751137553.f3ed23ac403772e0fe4ca9ee892544ec., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3ed23ac403772e0fe4ca9ee892544ec, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:59:10,016 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731751137553.f3ed23ac403772e0fe4ca9ee892544ec., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 because the exception is null or not the one we care about 2024-11-16T09:59:10,016 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3ed23ac403772e0fe4ca9ee892544ec, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T09:59:10,016 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] ipc.CallRunner(138): callId: 73 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34382 deadline: 1731751210013, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3ed23ac403772e0fe4ca9ee892544ec, server=037c58a19ddb,37267,1731751077844 2024-11-16T09:59:10,016 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3ed23ac403772e0fe4ca9ee892544ec, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T09:59:10,016 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] ipc.CallRunner(138): callId: 73 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34414 deadline: 1731751210013, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3ed23ac403772e0fe4ca9ee892544ec, server=037c58a19ddb,37267,1731751077844 2024-11-16T09:59:10,016 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731751137553.f3ed23ac403772e0fe4ca9ee892544ec., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 , the old value is region=TestAcidGuarantees,,1731751137553.f3ed23ac403772e0fe4ca9ee892544ec., hostname=037c58a19ddb,37267,1731751077844, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3ed23ac403772e0fe4ca9ee892544ec, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:59:10,016 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731751137553.f3ed23ac403772e0fe4ca9ee892544ec., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3ed23ac403772e0fe4ca9ee892544ec, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:59:10,016 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731751137553.f3ed23ac403772e0fe4ca9ee892544ec., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 because the exception is null or not the one we care about 2024-11-16T09:59:10,016 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731751137553.f3ed23ac403772e0fe4ca9ee892544ec., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 , the old value is region=TestAcidGuarantees,,1731751137553.f3ed23ac403772e0fe4ca9ee892544ec., hostname=037c58a19ddb,37267,1731751077844, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3ed23ac403772e0fe4ca9ee892544ec, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:59:10,016 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731751137553.f3ed23ac403772e0fe4ca9ee892544ec., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3ed23ac403772e0fe4ca9ee892544ec, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:59:10,016 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731751137553.f3ed23ac403772e0fe4ca9ee892544ec., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 because the exception is null or not the one we care about 2024-11-16T09:59:10,213 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3ed23ac403772e0fe4ca9ee892544ec, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T09:59:10,214 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] ipc.CallRunner(138): callId: 74 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34368 deadline: 1731751210213, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3ed23ac403772e0fe4ca9ee892544ec, server=037c58a19ddb,37267,1731751077844 2024-11-16T09:59:10,214 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3ed23ac403772e0fe4ca9ee892544ec, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T09:59:10,214 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731751137553.f3ed23ac403772e0fe4ca9ee892544ec., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 , the old value is region=TestAcidGuarantees,,1731751137553.f3ed23ac403772e0fe4ca9ee892544ec., hostname=037c58a19ddb,37267,1731751077844, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3ed23ac403772e0fe4ca9ee892544ec, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:59:10,214 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] ipc.CallRunner(138): callId: 73 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34408 deadline: 1731751210213, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3ed23ac403772e0fe4ca9ee892544ec, server=037c58a19ddb,37267,1731751077844 2024-11-16T09:59:10,214 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731751137553.f3ed23ac403772e0fe4ca9ee892544ec., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3ed23ac403772e0fe4ca9ee892544ec, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:59:10,214 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731751137553.f3ed23ac403772e0fe4ca9ee892544ec., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 because the exception is null or not the one we care about 2024-11-16T09:59:10,214 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731751137553.f3ed23ac403772e0fe4ca9ee892544ec., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 , the old value is region=TestAcidGuarantees,,1731751137553.f3ed23ac403772e0fe4ca9ee892544ec., hostname=037c58a19ddb,37267,1731751077844, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3ed23ac403772e0fe4ca9ee892544ec, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:59:10,214 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731751137553.f3ed23ac403772e0fe4ca9ee892544ec., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3ed23ac403772e0fe4ca9ee892544ec, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:59:10,214 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731751137553.f3ed23ac403772e0fe4ca9ee892544ec., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 because the exception is null or not the one we care about 2024-11-16T09:59:10,225 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3ed23ac403772e0fe4ca9ee892544ec, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T09:59:10,225 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] ipc.CallRunner(138): callId: 78 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34386 deadline: 1731751210223, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3ed23ac403772e0fe4ca9ee892544ec, server=037c58a19ddb,37267,1731751077844 2024-11-16T09:59:10,226 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3ed23ac403772e0fe4ca9ee892544ec, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T09:59:10,226 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731751137553.f3ed23ac403772e0fe4ca9ee892544ec., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 , the old value is region=TestAcidGuarantees,,1731751137553.f3ed23ac403772e0fe4ca9ee892544ec., hostname=037c58a19ddb,37267,1731751077844, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3ed23ac403772e0fe4ca9ee892544ec, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:59:10,226 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731751137553.f3ed23ac403772e0fe4ca9ee892544ec., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3ed23ac403772e0fe4ca9ee892544ec, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:59:10,226 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731751137553.f3ed23ac403772e0fe4ca9ee892544ec., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 because the exception is null or not the one we care about 2024-11-16T09:59:10,226 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] ipc.CallRunner(138): callId: 74 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34382 deadline: 1731751210223, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3ed23ac403772e0fe4ca9ee892544ec, server=037c58a19ddb,37267,1731751077844 2024-11-16T09:59:10,226 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3ed23ac403772e0fe4ca9ee892544ec, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T09:59:10,226 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731751137553.f3ed23ac403772e0fe4ca9ee892544ec., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 , the old value is region=TestAcidGuarantees,,1731751137553.f3ed23ac403772e0fe4ca9ee892544ec., hostname=037c58a19ddb,37267,1731751077844, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3ed23ac403772e0fe4ca9ee892544ec, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:59:10,226 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] ipc.CallRunner(138): callId: 74 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34414 deadline: 1731751210223, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3ed23ac403772e0fe4ca9ee892544ec, server=037c58a19ddb,37267,1731751077844 2024-11-16T09:59:10,226 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731751137553.f3ed23ac403772e0fe4ca9ee892544ec., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3ed23ac403772e0fe4ca9ee892544ec, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:59:10,226 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731751137553.f3ed23ac403772e0fe4ca9ee892544ec., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 because the exception is null or not the one we care about 2024-11-16T09:59:10,226 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731751137553.f3ed23ac403772e0fe4ca9ee892544ec., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 , the old value is region=TestAcidGuarantees,,1731751137553.f3ed23ac403772e0fe4ca9ee892544ec., hostname=037c58a19ddb,37267,1731751077844, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3ed23ac403772e0fe4ca9ee892544ec, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:59:10,226 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731751137553.f3ed23ac403772e0fe4ca9ee892544ec., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3ed23ac403772e0fe4ca9ee892544ec, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:59:10,226 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731751137553.f3ed23ac403772e0fe4ca9ee892544ec., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 because the exception is null or not the one we care about 2024-11-16T09:59:10,286 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=215 (bloomFilter=true), to=hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/f3ed23ac403772e0fe4ca9ee892544ec/.tmp/A/e1f68d6740d84ab2b22314b6666d769e 2024-11-16T09:59:10,294 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/f3ed23ac403772e0fe4ca9ee892544ec/.tmp/B/44e03547f5ba453bb3ce37077e891369 is 50, key is test_row_0/B:col10/1731751149876/Put/seqid=0 2024-11-16T09:59:10,297 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37591 is added to blk_1073742119_1295 (size=12151) 2024-11-16T09:59:10,492 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37877 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=68 2024-11-16T09:59:10,493 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-16T09:59:10,493 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37877 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-16T09:59:10,495 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37877 {}] procedure2.ProcedureExecutor(1139): Stored pid=70, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=70, table=TestAcidGuarantees 2024-11-16T09:59:10,495 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37877 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=70 2024-11-16T09:59:10,496 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=70, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=70, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-16T09:59:10,497 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=70, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=70, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-16T09:59:10,497 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=71, ppid=70, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-16T09:59:10,526 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3ed23ac403772e0fe4ca9ee892544ec, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T09:59:10,526 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] ipc.CallRunner(138): callId: 75 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34368 deadline: 1731751210523, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3ed23ac403772e0fe4ca9ee892544ec, server=037c58a19ddb,37267,1731751077844 2024-11-16T09:59:10,526 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3ed23ac403772e0fe4ca9ee892544ec, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T09:59:10,526 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731751137553.f3ed23ac403772e0fe4ca9ee892544ec., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 , the old value is region=TestAcidGuarantees,,1731751137553.f3ed23ac403772e0fe4ca9ee892544ec., hostname=037c58a19ddb,37267,1731751077844, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3ed23ac403772e0fe4ca9ee892544ec, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:59:10,526 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731751137553.f3ed23ac403772e0fe4ca9ee892544ec., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3ed23ac403772e0fe4ca9ee892544ec, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:59:10,526 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] ipc.CallRunner(138): callId: 74 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34408 deadline: 1731751210523, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3ed23ac403772e0fe4ca9ee892544ec, server=037c58a19ddb,37267,1731751077844 2024-11-16T09:59:10,526 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731751137553.f3ed23ac403772e0fe4ca9ee892544ec., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 because the exception is null or not the one we care about 2024-11-16T09:59:10,527 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731751137553.f3ed23ac403772e0fe4ca9ee892544ec., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 , the old value is region=TestAcidGuarantees,,1731751137553.f3ed23ac403772e0fe4ca9ee892544ec., hostname=037c58a19ddb,37267,1731751077844, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3ed23ac403772e0fe4ca9ee892544ec, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:59:10,527 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731751137553.f3ed23ac403772e0fe4ca9ee892544ec., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3ed23ac403772e0fe4ca9ee892544ec, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:59:10,527 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731751137553.f3ed23ac403772e0fe4ca9ee892544ec., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 because the exception is null or not the one we care about 2024-11-16T09:59:10,536 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3ed23ac403772e0fe4ca9ee892544ec, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T09:59:10,536 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] ipc.CallRunner(138): callId: 79 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34386 deadline: 1731751210533, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3ed23ac403772e0fe4ca9ee892544ec, server=037c58a19ddb,37267,1731751077844 2024-11-16T09:59:10,536 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731751137553.f3ed23ac403772e0fe4ca9ee892544ec., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 , the old value is region=TestAcidGuarantees,,1731751137553.f3ed23ac403772e0fe4ca9ee892544ec., hostname=037c58a19ddb,37267,1731751077844, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3ed23ac403772e0fe4ca9ee892544ec, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:59:10,537 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731751137553.f3ed23ac403772e0fe4ca9ee892544ec., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3ed23ac403772e0fe4ca9ee892544ec, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:59:10,536 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3ed23ac403772e0fe4ca9ee892544ec, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T09:59:10,537 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731751137553.f3ed23ac403772e0fe4ca9ee892544ec., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 because the exception is null or not the one we care about 2024-11-16T09:59:10,537 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] ipc.CallRunner(138): callId: 75 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34382 deadline: 1731751210533, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3ed23ac403772e0fe4ca9ee892544ec, server=037c58a19ddb,37267,1731751077844 2024-11-16T09:59:10,537 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3ed23ac403772e0fe4ca9ee892544ec, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T09:59:10,537 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] ipc.CallRunner(138): callId: 75 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34414 deadline: 1731751210533, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3ed23ac403772e0fe4ca9ee892544ec, server=037c58a19ddb,37267,1731751077844 2024-11-16T09:59:10,537 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731751137553.f3ed23ac403772e0fe4ca9ee892544ec., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 , the old value is region=TestAcidGuarantees,,1731751137553.f3ed23ac403772e0fe4ca9ee892544ec., hostname=037c58a19ddb,37267,1731751077844, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3ed23ac403772e0fe4ca9ee892544ec, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:59:10,537 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731751137553.f3ed23ac403772e0fe4ca9ee892544ec., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3ed23ac403772e0fe4ca9ee892544ec, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:59:10,537 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731751137553.f3ed23ac403772e0fe4ca9ee892544ec., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 because the exception is null or not the one we care about 2024-11-16T09:59:10,537 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731751137553.f3ed23ac403772e0fe4ca9ee892544ec., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 , the old value is region=TestAcidGuarantees,,1731751137553.f3ed23ac403772e0fe4ca9ee892544ec., hostname=037c58a19ddb,37267,1731751077844, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3ed23ac403772e0fe4ca9ee892544ec, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:59:10,537 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731751137553.f3ed23ac403772e0fe4ca9ee892544ec., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3ed23ac403772e0fe4ca9ee892544ec, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:59:10,537 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731751137553.f3ed23ac403772e0fe4ca9ee892544ec., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 because the exception is null or not the one we care about 2024-11-16T09:59:10,603 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37877 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=70 2024-11-16T09:59:10,649 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=37267 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=71 2024-11-16T09:59:10,649 DEBUG [RS_FLUSH_OPERATIONS-regionserver/037c58a19ddb:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731751137553.f3ed23ac403772e0fe4ca9ee892544ec. 2024-11-16T09:59:10,650 DEBUG [RS_FLUSH_OPERATIONS-regionserver/037c58a19ddb:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731751137553.f3ed23ac403772e0fe4ca9ee892544ec. as already flushing 2024-11-16T09:59:10,650 DEBUG [RS_FLUSH_OPERATIONS-regionserver/037c58a19ddb:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731751137553.f3ed23ac403772e0fe4ca9ee892544ec. 2024-11-16T09:59:10,650 ERROR [RS_FLUSH_OPERATIONS-regionserver/037c58a19ddb:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] handler.RSProcedureHandler(58): pid=71 java.io.IOException: Unable to complete flush {ENCODED => f3ed23ac403772e0fe4ca9ee892544ec, NAME => 'TestAcidGuarantees,,1731751137553.f3ed23ac403772e0fe4ca9ee892544ec.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T09:59:10,650 DEBUG [RS_FLUSH_OPERATIONS-regionserver/037c58a19ddb:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=71 java.io.IOException: Unable to complete flush {ENCODED => f3ed23ac403772e0fe4ca9ee892544ec, NAME => 'TestAcidGuarantees,,1731751137553.f3ed23ac403772e0fe4ca9ee892544ec.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T09:59:10,650 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37877 {}] master.HMaster(4177): Remote procedure failed, pid=71 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => f3ed23ac403772e0fe4ca9ee892544ec, NAME => 'TestAcidGuarantees,,1731751137553.f3ed23ac403772e0fe4ca9ee892544ec.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => f3ed23ac403772e0fe4ca9ee892544ec, NAME => 'TestAcidGuarantees,,1731751137553.f3ed23ac403772e0fe4ca9ee892544ec.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T09:59:10,698 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=215 (bloomFilter=true), to=hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/f3ed23ac403772e0fe4ca9ee892544ec/.tmp/B/44e03547f5ba453bb3ce37077e891369 2024-11-16T09:59:10,705 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/f3ed23ac403772e0fe4ca9ee892544ec/.tmp/C/c7691ed6881640f39f20fbf39543222c is 50, key is test_row_0/C:col10/1731751149876/Put/seqid=0 2024-11-16T09:59:10,709 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37591 is added to blk_1073742120_1296 (size=12151) 2024-11-16T09:59:10,802 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=37267 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=71 2024-11-16T09:59:10,802 DEBUG [RS_FLUSH_OPERATIONS-regionserver/037c58a19ddb:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731751137553.f3ed23ac403772e0fe4ca9ee892544ec. 2024-11-16T09:59:10,803 DEBUG [RS_FLUSH_OPERATIONS-regionserver/037c58a19ddb:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731751137553.f3ed23ac403772e0fe4ca9ee892544ec. as already flushing 2024-11-16T09:59:10,803 DEBUG [RS_FLUSH_OPERATIONS-regionserver/037c58a19ddb:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731751137553.f3ed23ac403772e0fe4ca9ee892544ec. 2024-11-16T09:59:10,803 ERROR [RS_FLUSH_OPERATIONS-regionserver/037c58a19ddb:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] handler.RSProcedureHandler(58): pid=71 java.io.IOException: Unable to complete flush {ENCODED => f3ed23ac403772e0fe4ca9ee892544ec, NAME => 'TestAcidGuarantees,,1731751137553.f3ed23ac403772e0fe4ca9ee892544ec.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T09:59:10,803 DEBUG [RS_FLUSH_OPERATIONS-regionserver/037c58a19ddb:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=71 java.io.IOException: Unable to complete flush {ENCODED => f3ed23ac403772e0fe4ca9ee892544ec, NAME => 'TestAcidGuarantees,,1731751137553.f3ed23ac403772e0fe4ca9ee892544ec.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T09:59:10,804 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37877 {}] master.HMaster(4177): Remote procedure failed, pid=71 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => f3ed23ac403772e0fe4ca9ee892544ec, NAME => 'TestAcidGuarantees,,1731751137553.f3ed23ac403772e0fe4ca9ee892544ec.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => f3ed23ac403772e0fe4ca9ee892544ec, NAME => 'TestAcidGuarantees,,1731751137553.f3ed23ac403772e0fe4ca9ee892544ec.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T09:59:10,813 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37877 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=70 2024-11-16T09:59:10,956 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=37267 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=71 2024-11-16T09:59:10,956 DEBUG [RS_FLUSH_OPERATIONS-regionserver/037c58a19ddb:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731751137553.f3ed23ac403772e0fe4ca9ee892544ec. 2024-11-16T09:59:10,956 DEBUG [RS_FLUSH_OPERATIONS-regionserver/037c58a19ddb:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731751137553.f3ed23ac403772e0fe4ca9ee892544ec. as already flushing 2024-11-16T09:59:10,956 DEBUG [RS_FLUSH_OPERATIONS-regionserver/037c58a19ddb:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731751137553.f3ed23ac403772e0fe4ca9ee892544ec. 2024-11-16T09:59:10,956 ERROR [RS_FLUSH_OPERATIONS-regionserver/037c58a19ddb:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] handler.RSProcedureHandler(58): pid=71 java.io.IOException: Unable to complete flush {ENCODED => f3ed23ac403772e0fe4ca9ee892544ec, NAME => 'TestAcidGuarantees,,1731751137553.f3ed23ac403772e0fe4ca9ee892544ec.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T09:59:10,956 DEBUG [RS_FLUSH_OPERATIONS-regionserver/037c58a19ddb:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=71 java.io.IOException: Unable to complete flush {ENCODED => f3ed23ac403772e0fe4ca9ee892544ec, NAME => 'TestAcidGuarantees,,1731751137553.f3ed23ac403772e0fe4ca9ee892544ec.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T09:59:10,957 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37877 {}] master.HMaster(4177): Remote procedure failed, pid=71 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => f3ed23ac403772e0fe4ca9ee892544ec, NAME => 'TestAcidGuarantees,,1731751137553.f3ed23ac403772e0fe4ca9ee892544ec.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => f3ed23ac403772e0fe4ca9ee892544ec, NAME => 'TestAcidGuarantees,,1731751137553.f3ed23ac403772e0fe4ca9ee892544ec.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T09:59:11,035 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3ed23ac403772e0fe4ca9ee892544ec, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T09:59:11,035 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] ipc.CallRunner(138): callId: 76 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34368 deadline: 1731751211032, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3ed23ac403772e0fe4ca9ee892544ec, server=037c58a19ddb,37267,1731751077844 2024-11-16T09:59:11,035 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3ed23ac403772e0fe4ca9ee892544ec, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T09:59:11,035 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] ipc.CallRunner(138): callId: 75 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34408 deadline: 1731751211032, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3ed23ac403772e0fe4ca9ee892544ec, server=037c58a19ddb,37267,1731751077844 2024-11-16T09:59:11,035 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731751137553.f3ed23ac403772e0fe4ca9ee892544ec., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 , the old value is region=TestAcidGuarantees,,1731751137553.f3ed23ac403772e0fe4ca9ee892544ec., hostname=037c58a19ddb,37267,1731751077844, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3ed23ac403772e0fe4ca9ee892544ec, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:59:11,035 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731751137553.f3ed23ac403772e0fe4ca9ee892544ec., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3ed23ac403772e0fe4ca9ee892544ec, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:59:11,035 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731751137553.f3ed23ac403772e0fe4ca9ee892544ec., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 because the exception is null or not the one we care about 2024-11-16T09:59:11,035 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731751137553.f3ed23ac403772e0fe4ca9ee892544ec., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 , the old value is region=TestAcidGuarantees,,1731751137553.f3ed23ac403772e0fe4ca9ee892544ec., hostname=037c58a19ddb,37267,1731751077844, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3ed23ac403772e0fe4ca9ee892544ec, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:59:11,036 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731751137553.f3ed23ac403772e0fe4ca9ee892544ec., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3ed23ac403772e0fe4ca9ee892544ec, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:59:11,036 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731751137553.f3ed23ac403772e0fe4ca9ee892544ec., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 because the exception is null or not the one we care about 2024-11-16T09:59:11,045 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3ed23ac403772e0fe4ca9ee892544ec, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T09:59:11,045 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] ipc.CallRunner(138): callId: 80 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34386 deadline: 1731751211042, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3ed23ac403772e0fe4ca9ee892544ec, server=037c58a19ddb,37267,1731751077844 2024-11-16T09:59:11,046 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3ed23ac403772e0fe4ca9ee892544ec, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T09:59:11,046 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731751137553.f3ed23ac403772e0fe4ca9ee892544ec., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 , the old value is region=TestAcidGuarantees,,1731751137553.f3ed23ac403772e0fe4ca9ee892544ec., hostname=037c58a19ddb,37267,1731751077844, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3ed23ac403772e0fe4ca9ee892544ec, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:59:11,046 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731751137553.f3ed23ac403772e0fe4ca9ee892544ec., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3ed23ac403772e0fe4ca9ee892544ec, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:59:11,046 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731751137553.f3ed23ac403772e0fe4ca9ee892544ec., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 because the exception is null or not the one we care about 2024-11-16T09:59:11,046 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] ipc.CallRunner(138): callId: 76 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34382 deadline: 1731751211043, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3ed23ac403772e0fe4ca9ee892544ec, server=037c58a19ddb,37267,1731751077844 2024-11-16T09:59:11,046 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3ed23ac403772e0fe4ca9ee892544ec, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T09:59:11,046 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731751137553.f3ed23ac403772e0fe4ca9ee892544ec., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 , the old value is region=TestAcidGuarantees,,1731751137553.f3ed23ac403772e0fe4ca9ee892544ec., hostname=037c58a19ddb,37267,1731751077844, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3ed23ac403772e0fe4ca9ee892544ec, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:59:11,046 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731751137553.f3ed23ac403772e0fe4ca9ee892544ec., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3ed23ac403772e0fe4ca9ee892544ec, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:59:11,046 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] ipc.CallRunner(138): callId: 76 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34414 deadline: 1731751211043, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3ed23ac403772e0fe4ca9ee892544ec, server=037c58a19ddb,37267,1731751077844 2024-11-16T09:59:11,046 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731751137553.f3ed23ac403772e0fe4ca9ee892544ec., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 because the exception is null or not the one we care about 2024-11-16T09:59:11,046 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731751137553.f3ed23ac403772e0fe4ca9ee892544ec., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 , the old value is region=TestAcidGuarantees,,1731751137553.f3ed23ac403772e0fe4ca9ee892544ec., hostname=037c58a19ddb,37267,1731751077844, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3ed23ac403772e0fe4ca9ee892544ec, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:59:11,046 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731751137553.f3ed23ac403772e0fe4ca9ee892544ec., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3ed23ac403772e0fe4ca9ee892544ec, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:59:11,046 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731751137553.f3ed23ac403772e0fe4ca9ee892544ec., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 because the exception is null or not the one we care about 2024-11-16T09:59:11,109 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=37267 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=71 2024-11-16T09:59:11,109 DEBUG [RS_FLUSH_OPERATIONS-regionserver/037c58a19ddb:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731751137553.f3ed23ac403772e0fe4ca9ee892544ec. 2024-11-16T09:59:11,109 DEBUG [RS_FLUSH_OPERATIONS-regionserver/037c58a19ddb:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731751137553.f3ed23ac403772e0fe4ca9ee892544ec. as already flushing 2024-11-16T09:59:11,109 DEBUG [RS_FLUSH_OPERATIONS-regionserver/037c58a19ddb:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731751137553.f3ed23ac403772e0fe4ca9ee892544ec. 2024-11-16T09:59:11,109 ERROR [RS_FLUSH_OPERATIONS-regionserver/037c58a19ddb:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] handler.RSProcedureHandler(58): pid=71 java.io.IOException: Unable to complete flush {ENCODED => f3ed23ac403772e0fe4ca9ee892544ec, NAME => 'TestAcidGuarantees,,1731751137553.f3ed23ac403772e0fe4ca9ee892544ec.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T09:59:11,110 DEBUG [RS_FLUSH_OPERATIONS-regionserver/037c58a19ddb:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=71 java.io.IOException: Unable to complete flush {ENCODED => f3ed23ac403772e0fe4ca9ee892544ec, NAME => 'TestAcidGuarantees,,1731751137553.f3ed23ac403772e0fe4ca9ee892544ec.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T09:59:11,110 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=215 (bloomFilter=true), to=hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/f3ed23ac403772e0fe4ca9ee892544ec/.tmp/C/c7691ed6881640f39f20fbf39543222c 2024-11-16T09:59:11,110 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37877 {}] master.HMaster(4177): Remote procedure failed, pid=71 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => f3ed23ac403772e0fe4ca9ee892544ec, NAME => 'TestAcidGuarantees,,1731751137553.f3ed23ac403772e0fe4ca9ee892544ec.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => f3ed23ac403772e0fe4ca9ee892544ec, NAME => 'TestAcidGuarantees,,1731751137553.f3ed23ac403772e0fe4ca9ee892544ec.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T09:59:11,115 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/f3ed23ac403772e0fe4ca9ee892544ec/.tmp/A/e1f68d6740d84ab2b22314b6666d769e as hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/f3ed23ac403772e0fe4ca9ee892544ec/A/e1f68d6740d84ab2b22314b6666d769e 2024-11-16T09:59:11,119 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/f3ed23ac403772e0fe4ca9ee892544ec/A/e1f68d6740d84ab2b22314b6666d769e, entries=200, sequenceid=215, filesize=14.2 K 2024-11-16T09:59:11,120 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/f3ed23ac403772e0fe4ca9ee892544ec/.tmp/B/44e03547f5ba453bb3ce37077e891369 as hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/f3ed23ac403772e0fe4ca9ee892544ec/B/44e03547f5ba453bb3ce37077e891369 2024-11-16T09:59:11,123 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37877 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=70 2024-11-16T09:59:11,124 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/f3ed23ac403772e0fe4ca9ee892544ec/B/44e03547f5ba453bb3ce37077e891369, entries=150, sequenceid=215, filesize=11.9 K 2024-11-16T09:59:11,125 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/f3ed23ac403772e0fe4ca9ee892544ec/.tmp/C/c7691ed6881640f39f20fbf39543222c as hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/f3ed23ac403772e0fe4ca9ee892544ec/C/c7691ed6881640f39f20fbf39543222c 2024-11-16T09:59:11,129 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/f3ed23ac403772e0fe4ca9ee892544ec/C/c7691ed6881640f39f20fbf39543222c, entries=150, sequenceid=215, filesize=11.9 K 2024-11-16T09:59:11,129 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~93.93 KB/96180, heapSize ~246.80 KB/252720, currentSize=107.34 KB/109920 for f3ed23ac403772e0fe4ca9ee892544ec in 1253ms, sequenceid=215, compaction requested=true 2024-11-16T09:59:11,129 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for f3ed23ac403772e0fe4ca9ee892544ec: 2024-11-16T09:59:11,130 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store f3ed23ac403772e0fe4ca9ee892544ec:A, priority=-2147483648, current under compaction store size is 1 2024-11-16T09:59:11,130 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-16T09:59:11,130 DEBUG [RS:0;037c58a19ddb:37267-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-16T09:59:11,130 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store f3ed23ac403772e0fe4ca9ee892544ec:B, priority=-2147483648, current under compaction store size is 2 2024-11-16T09:59:11,130 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-16T09:59:11,130 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store f3ed23ac403772e0fe4ca9ee892544ec:C, priority=-2147483648, current under compaction store size is 3 2024-11-16T09:59:11,130 DEBUG [RS:0;037c58a19ddb:37267-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-16T09:59:11,130 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-16T09:59:11,131 DEBUG [RS:0;037c58a19ddb:37267-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 51336 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-16T09:59:11,131 DEBUG [RS:0;037c58a19ddb:37267-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48946 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-16T09:59:11,131 DEBUG [RS:0;037c58a19ddb:37267-longCompactions-0 {}] regionserver.HStore(1541): f3ed23ac403772e0fe4ca9ee892544ec/B is initiating minor compaction (all files) 2024-11-16T09:59:11,131 DEBUG [RS:0;037c58a19ddb:37267-shortCompactions-0 {}] regionserver.HStore(1541): f3ed23ac403772e0fe4ca9ee892544ec/A is initiating minor compaction (all files) 2024-11-16T09:59:11,131 INFO [RS:0;037c58a19ddb:37267-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of f3ed23ac403772e0fe4ca9ee892544ec/B in TestAcidGuarantees,,1731751137553.f3ed23ac403772e0fe4ca9ee892544ec. 2024-11-16T09:59:11,131 INFO [RS:0;037c58a19ddb:37267-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of f3ed23ac403772e0fe4ca9ee892544ec/A in TestAcidGuarantees,,1731751137553.f3ed23ac403772e0fe4ca9ee892544ec. 2024-11-16T09:59:11,131 INFO [RS:0;037c58a19ddb:37267-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/f3ed23ac403772e0fe4ca9ee892544ec/B/59e77647903f4960ababcd7f53bcaae7, hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/f3ed23ac403772e0fe4ca9ee892544ec/B/192870787ef94f548016d8968e2641c5, hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/f3ed23ac403772e0fe4ca9ee892544ec/B/97437b43d51e496ba02e0a3029c174a6, hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/f3ed23ac403772e0fe4ca9ee892544ec/B/44e03547f5ba453bb3ce37077e891369] into tmpdir=hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/f3ed23ac403772e0fe4ca9ee892544ec/.tmp, totalSize=47.8 K 2024-11-16T09:59:11,131 INFO [RS:0;037c58a19ddb:37267-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/f3ed23ac403772e0fe4ca9ee892544ec/A/f9b5d8015d1b4e76a61f9ad1279908b4, hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/f3ed23ac403772e0fe4ca9ee892544ec/A/726ccf100eb646049bd2e286da649ab2, hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/f3ed23ac403772e0fe4ca9ee892544ec/A/2a41a1f803ea4e6e9929313e235b0236, hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/f3ed23ac403772e0fe4ca9ee892544ec/A/e1f68d6740d84ab2b22314b6666d769e] into tmpdir=hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/f3ed23ac403772e0fe4ca9ee892544ec/.tmp, totalSize=50.1 K 2024-11-16T09:59:11,131 DEBUG [RS:0;037c58a19ddb:37267-shortCompactions-0 {}] compactions.Compactor(225): Compacting f9b5d8015d1b4e76a61f9ad1279908b4, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=159, earliestPutTs=1731751146251 2024-11-16T09:59:11,131 DEBUG [RS:0;037c58a19ddb:37267-longCompactions-0 {}] compactions.Compactor(225): Compacting 59e77647903f4960ababcd7f53bcaae7, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=159, earliestPutTs=1731751146251 2024-11-16T09:59:11,132 DEBUG [RS:0;037c58a19ddb:37267-longCompactions-0 {}] compactions.Compactor(225): Compacting 192870787ef94f548016d8968e2641c5, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=173, earliestPutTs=1731751148034 2024-11-16T09:59:11,132 DEBUG [RS:0;037c58a19ddb:37267-shortCompactions-0 {}] compactions.Compactor(225): Compacting 726ccf100eb646049bd2e286da649ab2, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=173, earliestPutTs=1731751148034 2024-11-16T09:59:11,132 DEBUG [RS:0;037c58a19ddb:37267-longCompactions-0 {}] compactions.Compactor(225): Compacting 97437b43d51e496ba02e0a3029c174a6, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=198, earliestPutTs=1731751148059 2024-11-16T09:59:11,132 DEBUG [RS:0;037c58a19ddb:37267-shortCompactions-0 {}] compactions.Compactor(225): Compacting 2a41a1f803ea4e6e9929313e235b0236, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=198, earliestPutTs=1731751148059 2024-11-16T09:59:11,132 DEBUG [RS:0;037c58a19ddb:37267-shortCompactions-0 {}] compactions.Compactor(225): Compacting e1f68d6740d84ab2b22314b6666d769e, keycount=200, bloomtype=ROW, size=14.2 K, encoding=NONE, compression=NONE, seqNum=215, earliestPutTs=1731751149875 2024-11-16T09:59:11,133 DEBUG [RS:0;037c58a19ddb:37267-longCompactions-0 {}] compactions.Compactor(225): Compacting 44e03547f5ba453bb3ce37077e891369, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=215, earliestPutTs=1731751149875 2024-11-16T09:59:11,158 INFO [RS:0;037c58a19ddb:37267-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): f3ed23ac403772e0fe4ca9ee892544ec#A#compaction#252 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-16T09:59:11,159 INFO [RS:0;037c58a19ddb:37267-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): f3ed23ac403772e0fe4ca9ee892544ec#B#compaction#253 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-16T09:59:11,159 DEBUG [RS:0;037c58a19ddb:37267-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/f3ed23ac403772e0fe4ca9ee892544ec/.tmp/A/1192a667db6a468d8f92ad86a890bac9 is 50, key is test_row_0/A:col10/1731751149876/Put/seqid=0 2024-11-16T09:59:11,159 DEBUG [RS:0;037c58a19ddb:37267-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/f3ed23ac403772e0fe4ca9ee892544ec/.tmp/B/5c3c932b27c348f68a5ec53b7746f224 is 50, key is test_row_0/B:col10/1731751149876/Put/seqid=0 2024-11-16T09:59:11,178 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37591 is added to blk_1073742122_1298 (size=12629) 2024-11-16T09:59:11,179 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37591 is added to blk_1073742121_1297 (size=12629) 2024-11-16T09:59:11,263 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=37267 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=71 2024-11-16T09:59:11,263 DEBUG [RS_FLUSH_OPERATIONS-regionserver/037c58a19ddb:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731751137553.f3ed23ac403772e0fe4ca9ee892544ec. 2024-11-16T09:59:11,263 INFO [RS_FLUSH_OPERATIONS-regionserver/037c58a19ddb:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HRegion(2902): Flushing f3ed23ac403772e0fe4ca9ee892544ec 3/3 column families, dataSize=107.34 KB heapSize=282 KB 2024-11-16T09:59:11,268 DEBUG [RS_FLUSH_OPERATIONS-regionserver/037c58a19ddb:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/f3ed23ac403772e0fe4ca9ee892544ec/.tmp/A/60896e91c5674529aaf3a78f98e92f39 is 50, key is test_row_0/A:col10/1731751149894/Put/seqid=0 2024-11-16T09:59:11,272 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37591 is added to blk_1073742123_1299 (size=12151) 2024-11-16T09:59:11,585 DEBUG [RS:0;037c58a19ddb:37267-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/f3ed23ac403772e0fe4ca9ee892544ec/.tmp/B/5c3c932b27c348f68a5ec53b7746f224 as hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/f3ed23ac403772e0fe4ca9ee892544ec/B/5c3c932b27c348f68a5ec53b7746f224 2024-11-16T09:59:11,585 DEBUG [RS:0;037c58a19ddb:37267-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/f3ed23ac403772e0fe4ca9ee892544ec/.tmp/A/1192a667db6a468d8f92ad86a890bac9 as hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/f3ed23ac403772e0fe4ca9ee892544ec/A/1192a667db6a468d8f92ad86a890bac9 2024-11-16T09:59:11,595 INFO [RS:0;037c58a19ddb:37267-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in f3ed23ac403772e0fe4ca9ee892544ec/A of f3ed23ac403772e0fe4ca9ee892544ec into 1192a667db6a468d8f92ad86a890bac9(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-16T09:59:11,595 INFO [RS:0;037c58a19ddb:37267-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in f3ed23ac403772e0fe4ca9ee892544ec/B of f3ed23ac403772e0fe4ca9ee892544ec into 5c3c932b27c348f68a5ec53b7746f224(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-16T09:59:11,595 DEBUG [RS:0;037c58a19ddb:37267-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for f3ed23ac403772e0fe4ca9ee892544ec: 2024-11-16T09:59:11,596 DEBUG [RS:0;037c58a19ddb:37267-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for f3ed23ac403772e0fe4ca9ee892544ec: 2024-11-16T09:59:11,596 INFO [RS:0;037c58a19ddb:37267-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731751137553.f3ed23ac403772e0fe4ca9ee892544ec., storeName=f3ed23ac403772e0fe4ca9ee892544ec/B, priority=12, startTime=1731751151130; duration=0sec 2024-11-16T09:59:11,596 INFO [RS:0;037c58a19ddb:37267-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731751137553.f3ed23ac403772e0fe4ca9ee892544ec., storeName=f3ed23ac403772e0fe4ca9ee892544ec/A, priority=12, startTime=1731751151130; duration=0sec 2024-11-16T09:59:11,596 DEBUG [RS:0;037c58a19ddb:37267-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-16T09:59:11,596 DEBUG [RS:0;037c58a19ddb:37267-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: f3ed23ac403772e0fe4ca9ee892544ec:A 2024-11-16T09:59:11,596 DEBUG [RS:0;037c58a19ddb:37267-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-16T09:59:11,597 DEBUG [RS:0;037c58a19ddb:37267-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48946 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-16T09:59:11,598 DEBUG [RS:0;037c58a19ddb:37267-shortCompactions-0 {}] regionserver.HStore(1541): f3ed23ac403772e0fe4ca9ee892544ec/C is initiating minor compaction (all files) 2024-11-16T09:59:11,598 INFO [RS:0;037c58a19ddb:37267-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of f3ed23ac403772e0fe4ca9ee892544ec/C in TestAcidGuarantees,,1731751137553.f3ed23ac403772e0fe4ca9ee892544ec. 2024-11-16T09:59:11,598 INFO [RS:0;037c58a19ddb:37267-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/f3ed23ac403772e0fe4ca9ee892544ec/C/98ad7c5b73d9470bbe63c231e0379a5a, hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/f3ed23ac403772e0fe4ca9ee892544ec/C/515f53734f6e4430b06393764cb56c0f, hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/f3ed23ac403772e0fe4ca9ee892544ec/C/a4edb180540e4bb7a2083c0f6df737f7, hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/f3ed23ac403772e0fe4ca9ee892544ec/C/c7691ed6881640f39f20fbf39543222c] into tmpdir=hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/f3ed23ac403772e0fe4ca9ee892544ec/.tmp, totalSize=47.8 K 2024-11-16T09:59:11,598 DEBUG [RS:0;037c58a19ddb:37267-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-16T09:59:11,598 DEBUG [RS:0;037c58a19ddb:37267-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: f3ed23ac403772e0fe4ca9ee892544ec:B 2024-11-16T09:59:11,598 DEBUG [RS:0;037c58a19ddb:37267-shortCompactions-0 {}] compactions.Compactor(225): Compacting 98ad7c5b73d9470bbe63c231e0379a5a, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=159, earliestPutTs=1731751146251 2024-11-16T09:59:11,599 DEBUG [RS:0;037c58a19ddb:37267-shortCompactions-0 {}] compactions.Compactor(225): Compacting 515f53734f6e4430b06393764cb56c0f, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=173, earliestPutTs=1731751148034 2024-11-16T09:59:11,599 DEBUG [RS:0;037c58a19ddb:37267-shortCompactions-0 {}] compactions.Compactor(225): Compacting a4edb180540e4bb7a2083c0f6df737f7, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=198, earliestPutTs=1731751148059 2024-11-16T09:59:11,599 DEBUG [RS:0;037c58a19ddb:37267-shortCompactions-0 {}] compactions.Compactor(225): Compacting c7691ed6881640f39f20fbf39543222c, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=215, earliestPutTs=1731751149875 2024-11-16T09:59:11,609 INFO [RS:0;037c58a19ddb:37267-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): f3ed23ac403772e0fe4ca9ee892544ec#C#compaction#255 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-16T09:59:11,609 DEBUG [RS:0;037c58a19ddb:37267-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/f3ed23ac403772e0fe4ca9ee892544ec/.tmp/C/cf880f53d4a14e80999862e85f6b78ee is 50, key is test_row_0/C:col10/1731751149876/Put/seqid=0 2024-11-16T09:59:11,614 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37591 is added to blk_1073742124_1300 (size=12629) 2024-11-16T09:59:11,633 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37877 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=70 2024-11-16T09:59:11,673 INFO [RS_FLUSH_OPERATIONS-regionserver/037c58a19ddb:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=234 (bloomFilter=true), to=hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/f3ed23ac403772e0fe4ca9ee892544ec/.tmp/A/60896e91c5674529aaf3a78f98e92f39 2024-11-16T09:59:11,681 DEBUG [RS_FLUSH_OPERATIONS-regionserver/037c58a19ddb:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/f3ed23ac403772e0fe4ca9ee892544ec/.tmp/B/71c6cade454748a2ab9b75ee1c0575bc is 50, key is test_row_0/B:col10/1731751149894/Put/seqid=0 2024-11-16T09:59:11,685 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37591 is added to blk_1073742125_1301 (size=12151) 2024-11-16T09:59:11,686 INFO [RS_FLUSH_OPERATIONS-regionserver/037c58a19ddb:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=234 (bloomFilter=true), to=hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/f3ed23ac403772e0fe4ca9ee892544ec/.tmp/B/71c6cade454748a2ab9b75ee1c0575bc 2024-11-16T09:59:11,695 DEBUG [RS_FLUSH_OPERATIONS-regionserver/037c58a19ddb:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/f3ed23ac403772e0fe4ca9ee892544ec/.tmp/C/e7becd72ceb849a3b38a042102b3a7a0 is 50, key is test_row_0/C:col10/1731751149894/Put/seqid=0 2024-11-16T09:59:11,699 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37591 is added to blk_1073742126_1302 (size=12151) 2024-11-16T09:59:11,699 INFO [RS_FLUSH_OPERATIONS-regionserver/037c58a19ddb:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=234 (bloomFilter=true), to=hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/f3ed23ac403772e0fe4ca9ee892544ec/.tmp/C/e7becd72ceb849a3b38a042102b3a7a0 2024-11-16T09:59:11,704 DEBUG [RS_FLUSH_OPERATIONS-regionserver/037c58a19ddb:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/f3ed23ac403772e0fe4ca9ee892544ec/.tmp/A/60896e91c5674529aaf3a78f98e92f39 as hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/f3ed23ac403772e0fe4ca9ee892544ec/A/60896e91c5674529aaf3a78f98e92f39 2024-11-16T09:59:11,709 INFO [RS_FLUSH_OPERATIONS-regionserver/037c58a19ddb:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/f3ed23ac403772e0fe4ca9ee892544ec/A/60896e91c5674529aaf3a78f98e92f39, entries=150, sequenceid=234, filesize=11.9 K 2024-11-16T09:59:11,710 DEBUG [RS_FLUSH_OPERATIONS-regionserver/037c58a19ddb:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/f3ed23ac403772e0fe4ca9ee892544ec/.tmp/B/71c6cade454748a2ab9b75ee1c0575bc as hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/f3ed23ac403772e0fe4ca9ee892544ec/B/71c6cade454748a2ab9b75ee1c0575bc 2024-11-16T09:59:11,717 INFO [RS_FLUSH_OPERATIONS-regionserver/037c58a19ddb:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/f3ed23ac403772e0fe4ca9ee892544ec/B/71c6cade454748a2ab9b75ee1c0575bc, entries=150, sequenceid=234, filesize=11.9 K 2024-11-16T09:59:11,718 DEBUG [RS_FLUSH_OPERATIONS-regionserver/037c58a19ddb:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/f3ed23ac403772e0fe4ca9ee892544ec/.tmp/C/e7becd72ceb849a3b38a042102b3a7a0 as hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/f3ed23ac403772e0fe4ca9ee892544ec/C/e7becd72ceb849a3b38a042102b3a7a0 2024-11-16T09:59:11,723 INFO [RS_FLUSH_OPERATIONS-regionserver/037c58a19ddb:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/f3ed23ac403772e0fe4ca9ee892544ec/C/e7becd72ceb849a3b38a042102b3a7a0, entries=150, sequenceid=234, filesize=11.9 K 2024-11-16T09:59:11,724 INFO [RS_FLUSH_OPERATIONS-regionserver/037c58a19ddb:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HRegion(3140): Finished flush of dataSize ~107.34 KB/109920, heapSize ~281.95 KB/288720, currentSize=0 B/0 for f3ed23ac403772e0fe4ca9ee892544ec in 460ms, sequenceid=234, compaction requested=false 2024-11-16T09:59:11,724 DEBUG [RS_FLUSH_OPERATIONS-regionserver/037c58a19ddb:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HRegion(2603): Flush status journal for f3ed23ac403772e0fe4ca9ee892544ec: 2024-11-16T09:59:11,724 DEBUG [RS_FLUSH_OPERATIONS-regionserver/037c58a19ddb:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731751137553.f3ed23ac403772e0fe4ca9ee892544ec. 2024-11-16T09:59:11,724 DEBUG [RS_FLUSH_OPERATIONS-regionserver/037c58a19ddb:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=71 2024-11-16T09:59:11,724 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37877 {}] master.HMaster(4169): Remote procedure done, pid=71 2024-11-16T09:59:11,728 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=71, resume processing ppid=70 2024-11-16T09:59:11,728 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=71, ppid=70, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.2280 sec 2024-11-16T09:59:11,730 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=70, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=70, table=TestAcidGuarantees in 1.2360 sec 2024-11-16T09:59:12,021 DEBUG [RS:0;037c58a19ddb:37267-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/f3ed23ac403772e0fe4ca9ee892544ec/.tmp/C/cf880f53d4a14e80999862e85f6b78ee as hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/f3ed23ac403772e0fe4ca9ee892544ec/C/cf880f53d4a14e80999862e85f6b78ee 2024-11-16T09:59:12,026 INFO [RS:0;037c58a19ddb:37267-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in f3ed23ac403772e0fe4ca9ee892544ec/C of f3ed23ac403772e0fe4ca9ee892544ec into cf880f53d4a14e80999862e85f6b78ee(size=12.3 K), total size for store is 24.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-16T09:59:12,026 DEBUG [RS:0;037c58a19ddb:37267-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for f3ed23ac403772e0fe4ca9ee892544ec: 2024-11-16T09:59:12,026 INFO [RS:0;037c58a19ddb:37267-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731751137553.f3ed23ac403772e0fe4ca9ee892544ec., storeName=f3ed23ac403772e0fe4ca9ee892544ec/C, priority=12, startTime=1731751151130; duration=0sec 2024-11-16T09:59:12,027 DEBUG [RS:0;037c58a19ddb:37267-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-16T09:59:12,027 DEBUG [RS:0;037c58a19ddb:37267-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: f3ed23ac403772e0fe4ca9ee892544ec:C 2024-11-16T09:59:12,052 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] regionserver.HRegion(8855): Flush requested on f3ed23ac403772e0fe4ca9ee892544ec 2024-11-16T09:59:12,052 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-16T09:59:12,052 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing f3ed23ac403772e0fe4ca9ee892544ec 3/3 column families, dataSize=53.67 KB heapSize=141.38 KB 2024-11-16T09:59:12,057 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/f3ed23ac403772e0fe4ca9ee892544ec/.tmp/A/8102fd38ad8549e2a0a14cc8eef8c537 is 50, key is test_row_0/A:col10/1731751152051/Put/seqid=0 2024-11-16T09:59:12,061 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37591 is added to blk_1073742127_1303 (size=12151) 2024-11-16T09:59:12,081 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3ed23ac403772e0fe4ca9ee892544ec, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T09:59:12,081 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] ipc.CallRunner(138): callId: 81 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34414 deadline: 1731751212077, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3ed23ac403772e0fe4ca9ee892544ec, server=037c58a19ddb,37267,1731751077844 2024-11-16T09:59:12,081 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731751137553.f3ed23ac403772e0fe4ca9ee892544ec., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 , the old value is region=TestAcidGuarantees,,1731751137553.f3ed23ac403772e0fe4ca9ee892544ec., hostname=037c58a19ddb,37267,1731751077844, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3ed23ac403772e0fe4ca9ee892544ec, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:59:12,081 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731751137553.f3ed23ac403772e0fe4ca9ee892544ec., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3ed23ac403772e0fe4ca9ee892544ec, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:59:12,082 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731751137553.f3ed23ac403772e0fe4ca9ee892544ec., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 because the exception is null or not the one we care about 2024-11-16T09:59:12,084 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3ed23ac403772e0fe4ca9ee892544ec, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T09:59:12,084 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] ipc.CallRunner(138): callId: 85 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34408 deadline: 1731751212079, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3ed23ac403772e0fe4ca9ee892544ec, server=037c58a19ddb,37267,1731751077844 2024-11-16T09:59:12,084 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731751137553.f3ed23ac403772e0fe4ca9ee892544ec., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 , the old value is region=TestAcidGuarantees,,1731751137553.f3ed23ac403772e0fe4ca9ee892544ec., hostname=037c58a19ddb,37267,1731751077844, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3ed23ac403772e0fe4ca9ee892544ec, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:59:12,084 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3ed23ac403772e0fe4ca9ee892544ec, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T09:59:12,084 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731751137553.f3ed23ac403772e0fe4ca9ee892544ec., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3ed23ac403772e0fe4ca9ee892544ec, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:59:12,084 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731751137553.f3ed23ac403772e0fe4ca9ee892544ec., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 because the exception is null or not the one we care about 2024-11-16T09:59:12,084 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] ipc.CallRunner(138): callId: 80 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34382 deadline: 1731751212080, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3ed23ac403772e0fe4ca9ee892544ec, server=037c58a19ddb,37267,1731751077844 2024-11-16T09:59:12,085 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731751137553.f3ed23ac403772e0fe4ca9ee892544ec., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 , the old value is region=TestAcidGuarantees,,1731751137553.f3ed23ac403772e0fe4ca9ee892544ec., hostname=037c58a19ddb,37267,1731751077844, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3ed23ac403772e0fe4ca9ee892544ec, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:59:12,085 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731751137553.f3ed23ac403772e0fe4ca9ee892544ec., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3ed23ac403772e0fe4ca9ee892544ec, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:59:12,085 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731751137553.f3ed23ac403772e0fe4ca9ee892544ec., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 because the exception is null or not the one we care about 2024-11-16T09:59:12,085 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3ed23ac403772e0fe4ca9ee892544ec, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T09:59:12,085 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] ipc.CallRunner(138): callId: 86 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34386 deadline: 1731751212081, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3ed23ac403772e0fe4ca9ee892544ec, server=037c58a19ddb,37267,1731751077844 2024-11-16T09:59:12,086 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731751137553.f3ed23ac403772e0fe4ca9ee892544ec., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 , the old value is region=TestAcidGuarantees,,1731751137553.f3ed23ac403772e0fe4ca9ee892544ec., hostname=037c58a19ddb,37267,1731751077844, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3ed23ac403772e0fe4ca9ee892544ec, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:59:12,086 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731751137553.f3ed23ac403772e0fe4ca9ee892544ec., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3ed23ac403772e0fe4ca9ee892544ec, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:59:12,086 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731751137553.f3ed23ac403772e0fe4ca9ee892544ec., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 because the exception is null or not the one we care about 2024-11-16T09:59:12,086 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3ed23ac403772e0fe4ca9ee892544ec, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T09:59:12,086 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] ipc.CallRunner(138): callId: 87 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34368 deadline: 1731751212082, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3ed23ac403772e0fe4ca9ee892544ec, server=037c58a19ddb,37267,1731751077844 2024-11-16T09:59:12,086 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731751137553.f3ed23ac403772e0fe4ca9ee892544ec., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 , the old value is region=TestAcidGuarantees,,1731751137553.f3ed23ac403772e0fe4ca9ee892544ec., hostname=037c58a19ddb,37267,1731751077844, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3ed23ac403772e0fe4ca9ee892544ec, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:59:12,086 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731751137553.f3ed23ac403772e0fe4ca9ee892544ec., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3ed23ac403772e0fe4ca9ee892544ec, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:59:12,086 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731751137553.f3ed23ac403772e0fe4ca9ee892544ec., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 because the exception is null or not the one we care about 2024-11-16T09:59:12,195 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3ed23ac403772e0fe4ca9ee892544ec, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T09:59:12,195 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] ipc.CallRunner(138): callId: 82 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34414 deadline: 1731751212193, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3ed23ac403772e0fe4ca9ee892544ec, server=037c58a19ddb,37267,1731751077844 2024-11-16T09:59:12,195 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731751137553.f3ed23ac403772e0fe4ca9ee892544ec., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 , the old value is region=TestAcidGuarantees,,1731751137553.f3ed23ac403772e0fe4ca9ee892544ec., hostname=037c58a19ddb,37267,1731751077844, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3ed23ac403772e0fe4ca9ee892544ec, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:59:12,195 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731751137553.f3ed23ac403772e0fe4ca9ee892544ec., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3ed23ac403772e0fe4ca9ee892544ec, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:59:12,195 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3ed23ac403772e0fe4ca9ee892544ec, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T09:59:12,195 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731751137553.f3ed23ac403772e0fe4ca9ee892544ec., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 because the exception is null or not the one we care about 2024-11-16T09:59:12,196 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] ipc.CallRunner(138): callId: 86 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34408 deadline: 1731751212193, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3ed23ac403772e0fe4ca9ee892544ec, server=037c58a19ddb,37267,1731751077844 2024-11-16T09:59:12,196 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3ed23ac403772e0fe4ca9ee892544ec, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T09:59:12,196 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731751137553.f3ed23ac403772e0fe4ca9ee892544ec., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 , the old value is region=TestAcidGuarantees,,1731751137553.f3ed23ac403772e0fe4ca9ee892544ec., hostname=037c58a19ddb,37267,1731751077844, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3ed23ac403772e0fe4ca9ee892544ec, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:59:12,196 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731751137553.f3ed23ac403772e0fe4ca9ee892544ec., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3ed23ac403772e0fe4ca9ee892544ec, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:59:12,196 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] ipc.CallRunner(138): callId: 81 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34382 deadline: 1731751212193, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3ed23ac403772e0fe4ca9ee892544ec, server=037c58a19ddb,37267,1731751077844 2024-11-16T09:59:12,196 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731751137553.f3ed23ac403772e0fe4ca9ee892544ec., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 because the exception is null or not the one we care about 2024-11-16T09:59:12,196 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731751137553.f3ed23ac403772e0fe4ca9ee892544ec., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 , the old value is region=TestAcidGuarantees,,1731751137553.f3ed23ac403772e0fe4ca9ee892544ec., hostname=037c58a19ddb,37267,1731751077844, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3ed23ac403772e0fe4ca9ee892544ec, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:59:12,196 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3ed23ac403772e0fe4ca9ee892544ec, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T09:59:12,196 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731751137553.f3ed23ac403772e0fe4ca9ee892544ec., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3ed23ac403772e0fe4ca9ee892544ec, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:59:12,196 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731751137553.f3ed23ac403772e0fe4ca9ee892544ec., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 because the exception is null or not the one we care about 2024-11-16T09:59:12,196 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] ipc.CallRunner(138): callId: 87 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34386 deadline: 1731751212193, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3ed23ac403772e0fe4ca9ee892544ec, server=037c58a19ddb,37267,1731751077844 2024-11-16T09:59:12,196 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3ed23ac403772e0fe4ca9ee892544ec, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T09:59:12,196 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731751137553.f3ed23ac403772e0fe4ca9ee892544ec., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 , the old value is region=TestAcidGuarantees,,1731751137553.f3ed23ac403772e0fe4ca9ee892544ec., hostname=037c58a19ddb,37267,1731751077844, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3ed23ac403772e0fe4ca9ee892544ec, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:59:12,196 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731751137553.f3ed23ac403772e0fe4ca9ee892544ec., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3ed23ac403772e0fe4ca9ee892544ec, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:59:12,196 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] ipc.CallRunner(138): callId: 88 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34368 deadline: 1731751212193, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3ed23ac403772e0fe4ca9ee892544ec, server=037c58a19ddb,37267,1731751077844 2024-11-16T09:59:12,196 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731751137553.f3ed23ac403772e0fe4ca9ee892544ec., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 because the exception is null or not the one we care about 2024-11-16T09:59:12,197 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731751137553.f3ed23ac403772e0fe4ca9ee892544ec., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 , the old value is region=TestAcidGuarantees,,1731751137553.f3ed23ac403772e0fe4ca9ee892544ec., hostname=037c58a19ddb,37267,1731751077844, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3ed23ac403772e0fe4ca9ee892544ec, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:59:12,197 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731751137553.f3ed23ac403772e0fe4ca9ee892544ec., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3ed23ac403772e0fe4ca9ee892544ec, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:59:12,197 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731751137553.f3ed23ac403772e0fe4ca9ee892544ec., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 because the exception is null or not the one we care about 2024-11-16T09:59:12,405 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3ed23ac403772e0fe4ca9ee892544ec, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T09:59:12,405 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] ipc.CallRunner(138): callId: 83 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34414 deadline: 1731751212403, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3ed23ac403772e0fe4ca9ee892544ec, server=037c58a19ddb,37267,1731751077844 2024-11-16T09:59:12,406 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731751137553.f3ed23ac403772e0fe4ca9ee892544ec., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 , the old value is region=TestAcidGuarantees,,1731751137553.f3ed23ac403772e0fe4ca9ee892544ec., hostname=037c58a19ddb,37267,1731751077844, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3ed23ac403772e0fe4ca9ee892544ec, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:59:12,405 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3ed23ac403772e0fe4ca9ee892544ec, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T09:59:12,406 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731751137553.f3ed23ac403772e0fe4ca9ee892544ec., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3ed23ac403772e0fe4ca9ee892544ec, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:59:12,406 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731751137553.f3ed23ac403772e0fe4ca9ee892544ec., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 because the exception is null or not the one we care about 2024-11-16T09:59:12,406 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] ipc.CallRunner(138): callId: 87 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34408 deadline: 1731751212403, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3ed23ac403772e0fe4ca9ee892544ec, server=037c58a19ddb,37267,1731751077844 2024-11-16T09:59:12,406 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3ed23ac403772e0fe4ca9ee892544ec, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T09:59:12,406 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731751137553.f3ed23ac403772e0fe4ca9ee892544ec., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 , the old value is region=TestAcidGuarantees,,1731751137553.f3ed23ac403772e0fe4ca9ee892544ec., hostname=037c58a19ddb,37267,1731751077844, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3ed23ac403772e0fe4ca9ee892544ec, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:59:12,406 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731751137553.f3ed23ac403772e0fe4ca9ee892544ec., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3ed23ac403772e0fe4ca9ee892544ec, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:59:12,406 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] ipc.CallRunner(138): callId: 88 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34386 deadline: 1731751212403, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3ed23ac403772e0fe4ca9ee892544ec, server=037c58a19ddb,37267,1731751077844 2024-11-16T09:59:12,406 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731751137553.f3ed23ac403772e0fe4ca9ee892544ec., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 because the exception is null or not the one we care about 2024-11-16T09:59:12,406 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3ed23ac403772e0fe4ca9ee892544ec, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T09:59:12,406 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] ipc.CallRunner(138): callId: 82 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34382 deadline: 1731751212403, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3ed23ac403772e0fe4ca9ee892544ec, server=037c58a19ddb,37267,1731751077844 2024-11-16T09:59:12,406 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3ed23ac403772e0fe4ca9ee892544ec, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T09:59:12,406 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731751137553.f3ed23ac403772e0fe4ca9ee892544ec., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 , the old value is region=TestAcidGuarantees,,1731751137553.f3ed23ac403772e0fe4ca9ee892544ec., hostname=037c58a19ddb,37267,1731751077844, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3ed23ac403772e0fe4ca9ee892544ec, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:59:12,406 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731751137553.f3ed23ac403772e0fe4ca9ee892544ec., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3ed23ac403772e0fe4ca9ee892544ec, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:59:12,406 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] ipc.CallRunner(138): callId: 89 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34368 deadline: 1731751212403, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3ed23ac403772e0fe4ca9ee892544ec, server=037c58a19ddb,37267,1731751077844 2024-11-16T09:59:12,406 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731751137553.f3ed23ac403772e0fe4ca9ee892544ec., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 because the exception is null or not the one we care about 2024-11-16T09:59:12,406 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731751137553.f3ed23ac403772e0fe4ca9ee892544ec., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 , the old value is region=TestAcidGuarantees,,1731751137553.f3ed23ac403772e0fe4ca9ee892544ec., hostname=037c58a19ddb,37267,1731751077844, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3ed23ac403772e0fe4ca9ee892544ec, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:59:12,406 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731751137553.f3ed23ac403772e0fe4ca9ee892544ec., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3ed23ac403772e0fe4ca9ee892544ec, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:59:12,406 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731751137553.f3ed23ac403772e0fe4ca9ee892544ec., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 because the exception is null or not the one we care about 2024-11-16T09:59:12,406 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731751137553.f3ed23ac403772e0fe4ca9ee892544ec., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 , the old value is region=TestAcidGuarantees,,1731751137553.f3ed23ac403772e0fe4ca9ee892544ec., hostname=037c58a19ddb,37267,1731751077844, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3ed23ac403772e0fe4ca9ee892544ec, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:59:12,407 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731751137553.f3ed23ac403772e0fe4ca9ee892544ec., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3ed23ac403772e0fe4ca9ee892544ec, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:59:12,407 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731751137553.f3ed23ac403772e0fe4ca9ee892544ec., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 because the exception is null or not the one we care about 2024-11-16T09:59:12,462 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=248 (bloomFilter=true), to=hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/f3ed23ac403772e0fe4ca9ee892544ec/.tmp/A/8102fd38ad8549e2a0a14cc8eef8c537 2024-11-16T09:59:12,469 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/f3ed23ac403772e0fe4ca9ee892544ec/.tmp/B/198169af9b9a4f33b34b10f090bf7c75 is 50, key is test_row_0/B:col10/1731751152051/Put/seqid=0 2024-11-16T09:59:12,473 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37591 is added to blk_1073742128_1304 (size=12151) 2024-11-16T09:59:12,473 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=248 (bloomFilter=true), to=hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/f3ed23ac403772e0fe4ca9ee892544ec/.tmp/B/198169af9b9a4f33b34b10f090bf7c75 2024-11-16T09:59:12,481 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/f3ed23ac403772e0fe4ca9ee892544ec/.tmp/C/d35b1dfd46a04514a8e15ae0d6e3ba67 is 50, key is test_row_0/C:col10/1731751152051/Put/seqid=0 2024-11-16T09:59:12,485 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37591 is added to blk_1073742129_1305 (size=12151) 2024-11-16T09:59:12,643 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37877 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=70 2024-11-16T09:59:12,643 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-16T09:59:12,644 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37877 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-16T09:59:12,645 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37877 {}] procedure2.ProcedureExecutor(1139): Stored pid=72, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=72, table=TestAcidGuarantees 2024-11-16T09:59:12,646 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37877 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=72 2024-11-16T09:59:12,646 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=72, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=72, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-16T09:59:12,647 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=72, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=72, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-16T09:59:12,647 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=73, ppid=72, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-16T09:59:12,716 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3ed23ac403772e0fe4ca9ee892544ec, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T09:59:12,716 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] ipc.CallRunner(138): callId: 84 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34414 deadline: 1731751212713, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3ed23ac403772e0fe4ca9ee892544ec, server=037c58a19ddb,37267,1731751077844 2024-11-16T09:59:12,716 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3ed23ac403772e0fe4ca9ee892544ec, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T09:59:12,716 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731751137553.f3ed23ac403772e0fe4ca9ee892544ec., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 , the old value is region=TestAcidGuarantees,,1731751137553.f3ed23ac403772e0fe4ca9ee892544ec., hostname=037c58a19ddb,37267,1731751077844, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3ed23ac403772e0fe4ca9ee892544ec, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:59:12,716 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] ipc.CallRunner(138): callId: 88 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34408 deadline: 1731751212713, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3ed23ac403772e0fe4ca9ee892544ec, server=037c58a19ddb,37267,1731751077844 2024-11-16T09:59:12,716 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731751137553.f3ed23ac403772e0fe4ca9ee892544ec., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3ed23ac403772e0fe4ca9ee892544ec, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:59:12,716 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731751137553.f3ed23ac403772e0fe4ca9ee892544ec., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 because the exception is null or not the one we care about 2024-11-16T09:59:12,716 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3ed23ac403772e0fe4ca9ee892544ec, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T09:59:12,716 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731751137553.f3ed23ac403772e0fe4ca9ee892544ec., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 , the old value is region=TestAcidGuarantees,,1731751137553.f3ed23ac403772e0fe4ca9ee892544ec., hostname=037c58a19ddb,37267,1731751077844, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3ed23ac403772e0fe4ca9ee892544ec, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:59:12,716 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] ipc.CallRunner(138): callId: 83 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34382 deadline: 1731751212713, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3ed23ac403772e0fe4ca9ee892544ec, server=037c58a19ddb,37267,1731751077844 2024-11-16T09:59:12,716 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731751137553.f3ed23ac403772e0fe4ca9ee892544ec., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3ed23ac403772e0fe4ca9ee892544ec, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:59:12,717 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731751137553.f3ed23ac403772e0fe4ca9ee892544ec., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 because the exception is null or not the one we care about 2024-11-16T09:59:12,717 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3ed23ac403772e0fe4ca9ee892544ec, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T09:59:12,717 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731751137553.f3ed23ac403772e0fe4ca9ee892544ec., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 , the old value is region=TestAcidGuarantees,,1731751137553.f3ed23ac403772e0fe4ca9ee892544ec., hostname=037c58a19ddb,37267,1731751077844, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3ed23ac403772e0fe4ca9ee892544ec, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:59:12,717 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731751137553.f3ed23ac403772e0fe4ca9ee892544ec., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3ed23ac403772e0fe4ca9ee892544ec, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:59:12,717 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] ipc.CallRunner(138): callId: 90 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34368 deadline: 1731751212713, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3ed23ac403772e0fe4ca9ee892544ec, server=037c58a19ddb,37267,1731751077844 2024-11-16T09:59:12,717 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731751137553.f3ed23ac403772e0fe4ca9ee892544ec., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 because the exception is null or not the one we care about 2024-11-16T09:59:12,717 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3ed23ac403772e0fe4ca9ee892544ec, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T09:59:12,717 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] ipc.CallRunner(138): callId: 89 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34386 deadline: 1731751212713, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3ed23ac403772e0fe4ca9ee892544ec, server=037c58a19ddb,37267,1731751077844 2024-11-16T09:59:12,717 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731751137553.f3ed23ac403772e0fe4ca9ee892544ec., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 , the old value is region=TestAcidGuarantees,,1731751137553.f3ed23ac403772e0fe4ca9ee892544ec., hostname=037c58a19ddb,37267,1731751077844, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3ed23ac403772e0fe4ca9ee892544ec, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:59:12,717 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731751137553.f3ed23ac403772e0fe4ca9ee892544ec., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3ed23ac403772e0fe4ca9ee892544ec, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:59:12,717 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731751137553.f3ed23ac403772e0fe4ca9ee892544ec., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 , the old value is region=TestAcidGuarantees,,1731751137553.f3ed23ac403772e0fe4ca9ee892544ec., hostname=037c58a19ddb,37267,1731751077844, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3ed23ac403772e0fe4ca9ee892544ec, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:59:12,717 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731751137553.f3ed23ac403772e0fe4ca9ee892544ec., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3ed23ac403772e0fe4ca9ee892544ec, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:59:12,717 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731751137553.f3ed23ac403772e0fe4ca9ee892544ec., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 because the exception is null or not the one we care about 2024-11-16T09:59:12,717 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731751137553.f3ed23ac403772e0fe4ca9ee892544ec., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 because the exception is null or not the one we care about 2024-11-16T09:59:12,752 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37877 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=72 2024-11-16T09:59:12,799 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=37267 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=73 2024-11-16T09:59:12,799 DEBUG [RS_FLUSH_OPERATIONS-regionserver/037c58a19ddb:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731751137553.f3ed23ac403772e0fe4ca9ee892544ec. 2024-11-16T09:59:12,800 DEBUG [RS_FLUSH_OPERATIONS-regionserver/037c58a19ddb:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731751137553.f3ed23ac403772e0fe4ca9ee892544ec. as already flushing 2024-11-16T09:59:12,800 DEBUG [RS_FLUSH_OPERATIONS-regionserver/037c58a19ddb:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731751137553.f3ed23ac403772e0fe4ca9ee892544ec. 2024-11-16T09:59:12,800 ERROR [RS_FLUSH_OPERATIONS-regionserver/037c58a19ddb:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] handler.RSProcedureHandler(58): pid=73 java.io.IOException: Unable to complete flush {ENCODED => f3ed23ac403772e0fe4ca9ee892544ec, NAME => 'TestAcidGuarantees,,1731751137553.f3ed23ac403772e0fe4ca9ee892544ec.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T09:59:12,800 DEBUG [RS_FLUSH_OPERATIONS-regionserver/037c58a19ddb:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=73 java.io.IOException: Unable to complete flush {ENCODED => f3ed23ac403772e0fe4ca9ee892544ec, NAME => 'TestAcidGuarantees,,1731751137553.f3ed23ac403772e0fe4ca9ee892544ec.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T09:59:12,801 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37877 {}] master.HMaster(4177): Remote procedure failed, pid=73 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => f3ed23ac403772e0fe4ca9ee892544ec, NAME => 'TestAcidGuarantees,,1731751137553.f3ed23ac403772e0fe4ca9ee892544ec.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => f3ed23ac403772e0fe4ca9ee892544ec, NAME => 'TestAcidGuarantees,,1731751137553.f3ed23ac403772e0fe4ca9ee892544ec.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T09:59:12,886 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=248 (bloomFilter=true), to=hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/f3ed23ac403772e0fe4ca9ee892544ec/.tmp/C/d35b1dfd46a04514a8e15ae0d6e3ba67 2024-11-16T09:59:12,890 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/f3ed23ac403772e0fe4ca9ee892544ec/.tmp/A/8102fd38ad8549e2a0a14cc8eef8c537 as hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/f3ed23ac403772e0fe4ca9ee892544ec/A/8102fd38ad8549e2a0a14cc8eef8c537 2024-11-16T09:59:12,894 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/f3ed23ac403772e0fe4ca9ee892544ec/A/8102fd38ad8549e2a0a14cc8eef8c537, entries=150, sequenceid=248, filesize=11.9 K 2024-11-16T09:59:12,895 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/f3ed23ac403772e0fe4ca9ee892544ec/.tmp/B/198169af9b9a4f33b34b10f090bf7c75 as hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/f3ed23ac403772e0fe4ca9ee892544ec/B/198169af9b9a4f33b34b10f090bf7c75 2024-11-16T09:59:12,899 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/f3ed23ac403772e0fe4ca9ee892544ec/B/198169af9b9a4f33b34b10f090bf7c75, entries=150, sequenceid=248, filesize=11.9 K 2024-11-16T09:59:12,899 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/f3ed23ac403772e0fe4ca9ee892544ec/.tmp/C/d35b1dfd46a04514a8e15ae0d6e3ba67 as hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/f3ed23ac403772e0fe4ca9ee892544ec/C/d35b1dfd46a04514a8e15ae0d6e3ba67 2024-11-16T09:59:12,903 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/f3ed23ac403772e0fe4ca9ee892544ec/C/d35b1dfd46a04514a8e15ae0d6e3ba67, entries=150, sequenceid=248, filesize=11.9 K 2024-11-16T09:59:12,904 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~141.33 KB/144720, currentSize=154.31 KB/158010 for f3ed23ac403772e0fe4ca9ee892544ec in 852ms, sequenceid=248, compaction requested=true 2024-11-16T09:59:12,904 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for f3ed23ac403772e0fe4ca9ee892544ec: 2024-11-16T09:59:12,905 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store f3ed23ac403772e0fe4ca9ee892544ec:A, priority=-2147483648, current under compaction store size is 1 2024-11-16T09:59:12,905 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-16T09:59:12,905 DEBUG [RS:0;037c58a19ddb:37267-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-16T09:59:12,905 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store f3ed23ac403772e0fe4ca9ee892544ec:B, priority=-2147483648, current under compaction store size is 2 2024-11-16T09:59:12,905 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-16T09:59:12,905 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store f3ed23ac403772e0fe4ca9ee892544ec:C, priority=-2147483648, current under compaction store size is 3 2024-11-16T09:59:12,905 DEBUG [RS:0;037c58a19ddb:37267-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-16T09:59:12,905 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-16T09:59:12,906 DEBUG [RS:0;037c58a19ddb:37267-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36931 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-16T09:59:12,906 DEBUG [RS:0;037c58a19ddb:37267-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36931 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-16T09:59:12,906 DEBUG [RS:0;037c58a19ddb:37267-shortCompactions-0 {}] regionserver.HStore(1541): f3ed23ac403772e0fe4ca9ee892544ec/A is initiating minor compaction (all files) 2024-11-16T09:59:12,906 DEBUG [RS:0;037c58a19ddb:37267-longCompactions-0 {}] regionserver.HStore(1541): f3ed23ac403772e0fe4ca9ee892544ec/B is initiating minor compaction (all files) 2024-11-16T09:59:12,906 INFO [RS:0;037c58a19ddb:37267-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of f3ed23ac403772e0fe4ca9ee892544ec/A in TestAcidGuarantees,,1731751137553.f3ed23ac403772e0fe4ca9ee892544ec. 2024-11-16T09:59:12,906 INFO [RS:0;037c58a19ddb:37267-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of f3ed23ac403772e0fe4ca9ee892544ec/B in TestAcidGuarantees,,1731751137553.f3ed23ac403772e0fe4ca9ee892544ec. 2024-11-16T09:59:12,906 INFO [RS:0;037c58a19ddb:37267-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/f3ed23ac403772e0fe4ca9ee892544ec/A/1192a667db6a468d8f92ad86a890bac9, hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/f3ed23ac403772e0fe4ca9ee892544ec/A/60896e91c5674529aaf3a78f98e92f39, hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/f3ed23ac403772e0fe4ca9ee892544ec/A/8102fd38ad8549e2a0a14cc8eef8c537] into tmpdir=hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/f3ed23ac403772e0fe4ca9ee892544ec/.tmp, totalSize=36.1 K 2024-11-16T09:59:12,906 INFO [RS:0;037c58a19ddb:37267-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/f3ed23ac403772e0fe4ca9ee892544ec/B/5c3c932b27c348f68a5ec53b7746f224, hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/f3ed23ac403772e0fe4ca9ee892544ec/B/71c6cade454748a2ab9b75ee1c0575bc, hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/f3ed23ac403772e0fe4ca9ee892544ec/B/198169af9b9a4f33b34b10f090bf7c75] into tmpdir=hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/f3ed23ac403772e0fe4ca9ee892544ec/.tmp, totalSize=36.1 K 2024-11-16T09:59:12,907 DEBUG [RS:0;037c58a19ddb:37267-shortCompactions-0 {}] compactions.Compactor(225): Compacting 1192a667db6a468d8f92ad86a890bac9, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=215, earliestPutTs=1731751149875 2024-11-16T09:59:12,907 DEBUG [RS:0;037c58a19ddb:37267-longCompactions-0 {}] compactions.Compactor(225): Compacting 5c3c932b27c348f68a5ec53b7746f224, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=215, earliestPutTs=1731751149875 2024-11-16T09:59:12,907 DEBUG [RS:0;037c58a19ddb:37267-longCompactions-0 {}] compactions.Compactor(225): Compacting 71c6cade454748a2ab9b75ee1c0575bc, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=234, earliestPutTs=1731751149894 2024-11-16T09:59:12,907 DEBUG [RS:0;037c58a19ddb:37267-shortCompactions-0 {}] compactions.Compactor(225): Compacting 60896e91c5674529aaf3a78f98e92f39, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=234, earliestPutTs=1731751149894 2024-11-16T09:59:12,908 DEBUG [RS:0;037c58a19ddb:37267-longCompactions-0 {}] compactions.Compactor(225): Compacting 198169af9b9a4f33b34b10f090bf7c75, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=248, earliestPutTs=1731751152043 2024-11-16T09:59:12,908 DEBUG [RS:0;037c58a19ddb:37267-shortCompactions-0 {}] compactions.Compactor(225): Compacting 8102fd38ad8549e2a0a14cc8eef8c537, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=248, earliestPutTs=1731751152043 2024-11-16T09:59:12,915 INFO [RS:0;037c58a19ddb:37267-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): f3ed23ac403772e0fe4ca9ee892544ec#A#compaction#261 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-16T09:59:12,916 DEBUG [RS:0;037c58a19ddb:37267-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/f3ed23ac403772e0fe4ca9ee892544ec/.tmp/A/1775c312f3b740ccb57738318487b1ee is 50, key is test_row_0/A:col10/1731751152051/Put/seqid=0 2024-11-16T09:59:12,917 INFO [RS:0;037c58a19ddb:37267-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): f3ed23ac403772e0fe4ca9ee892544ec#B#compaction#262 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-16T09:59:12,917 DEBUG [RS:0;037c58a19ddb:37267-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/f3ed23ac403772e0fe4ca9ee892544ec/.tmp/B/3df0f2796e9848a6af1729c89fb64a31 is 50, key is test_row_0/B:col10/1731751152051/Put/seqid=0 2024-11-16T09:59:12,920 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37591 is added to blk_1073742130_1306 (size=12731) 2024-11-16T09:59:12,924 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37591 is added to blk_1073742131_1307 (size=12731) 2024-11-16T09:59:12,953 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=37267 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=73 2024-11-16T09:59:12,953 DEBUG [RS_FLUSH_OPERATIONS-regionserver/037c58a19ddb:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731751137553.f3ed23ac403772e0fe4ca9ee892544ec. 2024-11-16T09:59:12,953 INFO [RS_FLUSH_OPERATIONS-regionserver/037c58a19ddb:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HRegion(2902): Flushing f3ed23ac403772e0fe4ca9ee892544ec 3/3 column families, dataSize=154.31 KB heapSize=405.05 KB 2024-11-16T09:59:12,957 DEBUG [RS_FLUSH_OPERATIONS-regionserver/037c58a19ddb:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/f3ed23ac403772e0fe4ca9ee892544ec/.tmp/A/882d0e38ac3f4f40bbb1b64db18c4a2d is 50, key is test_row_0/A:col10/1731751152080/Put/seqid=0 2024-11-16T09:59:12,961 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37591 is added to blk_1073742132_1308 (size=12301) 2024-11-16T09:59:12,963 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37877 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=72 2024-11-16T09:59:13,226 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] regionserver.HRegion(8855): Flush requested on f3ed23ac403772e0fe4ca9ee892544ec 2024-11-16T09:59:13,226 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731751137553.f3ed23ac403772e0fe4ca9ee892544ec. as already flushing 2024-11-16T09:59:13,232 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3ed23ac403772e0fe4ca9ee892544ec, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T09:59:13,233 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] ipc.CallRunner(138): callId: 92 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34368 deadline: 1731751213228, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3ed23ac403772e0fe4ca9ee892544ec, server=037c58a19ddb,37267,1731751077844 2024-11-16T09:59:13,233 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3ed23ac403772e0fe4ca9ee892544ec, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T09:59:13,233 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731751137553.f3ed23ac403772e0fe4ca9ee892544ec., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 , the old value is region=TestAcidGuarantees,,1731751137553.f3ed23ac403772e0fe4ca9ee892544ec., hostname=037c58a19ddb,37267,1731751077844, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3ed23ac403772e0fe4ca9ee892544ec, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:59:13,233 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731751137553.f3ed23ac403772e0fe4ca9ee892544ec., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3ed23ac403772e0fe4ca9ee892544ec, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:59:13,233 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] ipc.CallRunner(138): callId: 85 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34382 deadline: 1731751213228, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3ed23ac403772e0fe4ca9ee892544ec, server=037c58a19ddb,37267,1731751077844 2024-11-16T09:59:13,233 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731751137553.f3ed23ac403772e0fe4ca9ee892544ec., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 because the exception is null or not the one we care about 2024-11-16T09:59:13,233 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731751137553.f3ed23ac403772e0fe4ca9ee892544ec., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 , the old value is region=TestAcidGuarantees,,1731751137553.f3ed23ac403772e0fe4ca9ee892544ec., hostname=037c58a19ddb,37267,1731751077844, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3ed23ac403772e0fe4ca9ee892544ec, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:59:13,233 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3ed23ac403772e0fe4ca9ee892544ec, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T09:59:13,233 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731751137553.f3ed23ac403772e0fe4ca9ee892544ec., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3ed23ac403772e0fe4ca9ee892544ec, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:59:13,233 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731751137553.f3ed23ac403772e0fe4ca9ee892544ec., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 because the exception is null or not the one we care about 2024-11-16T09:59:13,233 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] ipc.CallRunner(138): callId: 91 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34386 deadline: 1731751213229, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3ed23ac403772e0fe4ca9ee892544ec, server=037c58a19ddb,37267,1731751077844 2024-11-16T09:59:13,234 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731751137553.f3ed23ac403772e0fe4ca9ee892544ec., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 , the old value is region=TestAcidGuarantees,,1731751137553.f3ed23ac403772e0fe4ca9ee892544ec., hostname=037c58a19ddb,37267,1731751077844, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3ed23ac403772e0fe4ca9ee892544ec, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:59:13,234 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731751137553.f3ed23ac403772e0fe4ca9ee892544ec., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3ed23ac403772e0fe4ca9ee892544ec, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:59:13,234 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731751137553.f3ed23ac403772e0fe4ca9ee892544ec., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 because the exception is null or not the one we care about 2024-11-16T09:59:13,235 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3ed23ac403772e0fe4ca9ee892544ec, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T09:59:13,235 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] ipc.CallRunner(138): callId: 87 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34414 deadline: 1731751213232, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3ed23ac403772e0fe4ca9ee892544ec, server=037c58a19ddb,37267,1731751077844 2024-11-16T09:59:13,235 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731751137553.f3ed23ac403772e0fe4ca9ee892544ec., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 , the old value is region=TestAcidGuarantees,,1731751137553.f3ed23ac403772e0fe4ca9ee892544ec., hostname=037c58a19ddb,37267,1731751077844, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3ed23ac403772e0fe4ca9ee892544ec, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:59:13,235 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731751137553.f3ed23ac403772e0fe4ca9ee892544ec., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3ed23ac403772e0fe4ca9ee892544ec, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:59:13,235 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731751137553.f3ed23ac403772e0fe4ca9ee892544ec., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 because the exception is null or not the one we care about 2024-11-16T09:59:13,236 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3ed23ac403772e0fe4ca9ee892544ec, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T09:59:13,236 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] ipc.CallRunner(138): callId: 91 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34408 deadline: 1731751213233, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3ed23ac403772e0fe4ca9ee892544ec, server=037c58a19ddb,37267,1731751077844 2024-11-16T09:59:13,237 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731751137553.f3ed23ac403772e0fe4ca9ee892544ec., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 , the old value is region=TestAcidGuarantees,,1731751137553.f3ed23ac403772e0fe4ca9ee892544ec., hostname=037c58a19ddb,37267,1731751077844, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3ed23ac403772e0fe4ca9ee892544ec, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:59:13,237 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731751137553.f3ed23ac403772e0fe4ca9ee892544ec., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3ed23ac403772e0fe4ca9ee892544ec, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:59:13,237 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731751137553.f3ed23ac403772e0fe4ca9ee892544ec., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 because the exception is null or not the one we care about 2024-11-16T09:59:13,273 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37877 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=72 2024-11-16T09:59:13,325 DEBUG [RS:0;037c58a19ddb:37267-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/f3ed23ac403772e0fe4ca9ee892544ec/.tmp/A/1775c312f3b740ccb57738318487b1ee as hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/f3ed23ac403772e0fe4ca9ee892544ec/A/1775c312f3b740ccb57738318487b1ee 2024-11-16T09:59:13,328 DEBUG [RS:0;037c58a19ddb:37267-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/f3ed23ac403772e0fe4ca9ee892544ec/.tmp/B/3df0f2796e9848a6af1729c89fb64a31 as hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/f3ed23ac403772e0fe4ca9ee892544ec/B/3df0f2796e9848a6af1729c89fb64a31 2024-11-16T09:59:13,330 INFO [RS:0;037c58a19ddb:37267-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in f3ed23ac403772e0fe4ca9ee892544ec/A of f3ed23ac403772e0fe4ca9ee892544ec into 1775c312f3b740ccb57738318487b1ee(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-16T09:59:13,331 DEBUG [RS:0;037c58a19ddb:37267-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for f3ed23ac403772e0fe4ca9ee892544ec: 2024-11-16T09:59:13,331 INFO [RS:0;037c58a19ddb:37267-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731751137553.f3ed23ac403772e0fe4ca9ee892544ec., storeName=f3ed23ac403772e0fe4ca9ee892544ec/A, priority=13, startTime=1731751152904; duration=0sec 2024-11-16T09:59:13,331 DEBUG [RS:0;037c58a19ddb:37267-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-16T09:59:13,331 DEBUG [RS:0;037c58a19ddb:37267-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: f3ed23ac403772e0fe4ca9ee892544ec:A 2024-11-16T09:59:13,331 DEBUG [RS:0;037c58a19ddb:37267-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-16T09:59:13,332 DEBUG [RS:0;037c58a19ddb:37267-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36931 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-16T09:59:13,332 DEBUG [RS:0;037c58a19ddb:37267-shortCompactions-0 {}] regionserver.HStore(1541): f3ed23ac403772e0fe4ca9ee892544ec/C is initiating minor compaction (all files) 2024-11-16T09:59:13,332 INFO [RS:0;037c58a19ddb:37267-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of f3ed23ac403772e0fe4ca9ee892544ec/C in TestAcidGuarantees,,1731751137553.f3ed23ac403772e0fe4ca9ee892544ec. 2024-11-16T09:59:13,332 INFO [RS:0;037c58a19ddb:37267-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/f3ed23ac403772e0fe4ca9ee892544ec/C/cf880f53d4a14e80999862e85f6b78ee, hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/f3ed23ac403772e0fe4ca9ee892544ec/C/e7becd72ceb849a3b38a042102b3a7a0, hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/f3ed23ac403772e0fe4ca9ee892544ec/C/d35b1dfd46a04514a8e15ae0d6e3ba67] into tmpdir=hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/f3ed23ac403772e0fe4ca9ee892544ec/.tmp, totalSize=36.1 K 2024-11-16T09:59:13,333 INFO [RS:0;037c58a19ddb:37267-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in f3ed23ac403772e0fe4ca9ee892544ec/B of f3ed23ac403772e0fe4ca9ee892544ec into 3df0f2796e9848a6af1729c89fb64a31(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-16T09:59:13,333 DEBUG [RS:0;037c58a19ddb:37267-shortCompactions-0 {}] compactions.Compactor(225): Compacting cf880f53d4a14e80999862e85f6b78ee, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=215, earliestPutTs=1731751149875 2024-11-16T09:59:13,333 DEBUG [RS:0;037c58a19ddb:37267-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for f3ed23ac403772e0fe4ca9ee892544ec: 2024-11-16T09:59:13,333 INFO [RS:0;037c58a19ddb:37267-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731751137553.f3ed23ac403772e0fe4ca9ee892544ec., storeName=f3ed23ac403772e0fe4ca9ee892544ec/B, priority=13, startTime=1731751152905; duration=0sec 2024-11-16T09:59:13,333 DEBUG [RS:0;037c58a19ddb:37267-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-16T09:59:13,333 DEBUG [RS:0;037c58a19ddb:37267-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: f3ed23ac403772e0fe4ca9ee892544ec:B 2024-11-16T09:59:13,333 DEBUG [RS:0;037c58a19ddb:37267-shortCompactions-0 {}] compactions.Compactor(225): Compacting e7becd72ceb849a3b38a042102b3a7a0, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=234, earliestPutTs=1731751149894 2024-11-16T09:59:13,334 DEBUG [RS:0;037c58a19ddb:37267-shortCompactions-0 {}] compactions.Compactor(225): Compacting d35b1dfd46a04514a8e15ae0d6e3ba67, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=248, earliestPutTs=1731751152043 2024-11-16T09:59:13,339 INFO [RS:0;037c58a19ddb:37267-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): f3ed23ac403772e0fe4ca9ee892544ec#C#compaction#264 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-16T09:59:13,340 DEBUG [RS:0;037c58a19ddb:37267-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/f3ed23ac403772e0fe4ca9ee892544ec/.tmp/C/1278f4f12e514b53ba17c59c2c5fa75c is 50, key is test_row_0/C:col10/1731751152051/Put/seqid=0 2024-11-16T09:59:13,343 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37591 is added to blk_1073742133_1309 (size=12731) 2024-11-16T09:59:13,343 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3ed23ac403772e0fe4ca9ee892544ec, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T09:59:13,344 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] ipc.CallRunner(138): callId: 93 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34368 deadline: 1731751213343, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3ed23ac403772e0fe4ca9ee892544ec, server=037c58a19ddb,37267,1731751077844 2024-11-16T09:59:13,344 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731751137553.f3ed23ac403772e0fe4ca9ee892544ec., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 , the old value is region=TestAcidGuarantees,,1731751137553.f3ed23ac403772e0fe4ca9ee892544ec., hostname=037c58a19ddb,37267,1731751077844, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3ed23ac403772e0fe4ca9ee892544ec, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:59:13,344 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3ed23ac403772e0fe4ca9ee892544ec, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T09:59:13,344 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731751137553.f3ed23ac403772e0fe4ca9ee892544ec., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3ed23ac403772e0fe4ca9ee892544ec, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:59:13,344 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731751137553.f3ed23ac403772e0fe4ca9ee892544ec., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 because the exception is null or not the one we care about 2024-11-16T09:59:13,344 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] ipc.CallRunner(138): callId: 92 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34408 deadline: 1731751213343, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3ed23ac403772e0fe4ca9ee892544ec, server=037c58a19ddb,37267,1731751077844 2024-11-16T09:59:13,344 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731751137553.f3ed23ac403772e0fe4ca9ee892544ec., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 , the old value is region=TestAcidGuarantees,,1731751137553.f3ed23ac403772e0fe4ca9ee892544ec., hostname=037c58a19ddb,37267,1731751077844, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3ed23ac403772e0fe4ca9ee892544ec, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:59:13,344 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731751137553.f3ed23ac403772e0fe4ca9ee892544ec., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3ed23ac403772e0fe4ca9ee892544ec, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:59:13,344 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731751137553.f3ed23ac403772e0fe4ca9ee892544ec., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 because the exception is null or not the one we care about 2024-11-16T09:59:13,345 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3ed23ac403772e0fe4ca9ee892544ec, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T09:59:13,346 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] ipc.CallRunner(138): callId: 86 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34382 deadline: 1731751213343, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3ed23ac403772e0fe4ca9ee892544ec, server=037c58a19ddb,37267,1731751077844 2024-11-16T09:59:13,346 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731751137553.f3ed23ac403772e0fe4ca9ee892544ec., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 , the old value is region=TestAcidGuarantees,,1731751137553.f3ed23ac403772e0fe4ca9ee892544ec., hostname=037c58a19ddb,37267,1731751077844, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3ed23ac403772e0fe4ca9ee892544ec, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:59:13,346 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731751137553.f3ed23ac403772e0fe4ca9ee892544ec., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3ed23ac403772e0fe4ca9ee892544ec, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:59:13,346 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3ed23ac403772e0fe4ca9ee892544ec, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T09:59:13,346 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731751137553.f3ed23ac403772e0fe4ca9ee892544ec., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 because the exception is null or not the one we care about 2024-11-16T09:59:13,346 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] ipc.CallRunner(138): callId: 88 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34414 deadline: 1731751213343, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3ed23ac403772e0fe4ca9ee892544ec, server=037c58a19ddb,37267,1731751077844 2024-11-16T09:59:13,346 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731751137553.f3ed23ac403772e0fe4ca9ee892544ec., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 , the old value is region=TestAcidGuarantees,,1731751137553.f3ed23ac403772e0fe4ca9ee892544ec., hostname=037c58a19ddb,37267,1731751077844, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3ed23ac403772e0fe4ca9ee892544ec, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:59:13,346 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731751137553.f3ed23ac403772e0fe4ca9ee892544ec., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3ed23ac403772e0fe4ca9ee892544ec, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:59:13,346 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731751137553.f3ed23ac403772e0fe4ca9ee892544ec., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 because the exception is null or not the one we care about 2024-11-16T09:59:13,346 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3ed23ac403772e0fe4ca9ee892544ec, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T09:59:13,346 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] ipc.CallRunner(138): callId: 92 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34386 deadline: 1731751213344, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3ed23ac403772e0fe4ca9ee892544ec, server=037c58a19ddb,37267,1731751077844 2024-11-16T09:59:13,347 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731751137553.f3ed23ac403772e0fe4ca9ee892544ec., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 , the old value is region=TestAcidGuarantees,,1731751137553.f3ed23ac403772e0fe4ca9ee892544ec., hostname=037c58a19ddb,37267,1731751077844, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3ed23ac403772e0fe4ca9ee892544ec, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:59:13,347 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731751137553.f3ed23ac403772e0fe4ca9ee892544ec., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3ed23ac403772e0fe4ca9ee892544ec, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:59:13,347 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731751137553.f3ed23ac403772e0fe4ca9ee892544ec., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 because the exception is null or not the one we care about 2024-11-16T09:59:13,349 DEBUG [RS:0;037c58a19ddb:37267-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/f3ed23ac403772e0fe4ca9ee892544ec/.tmp/C/1278f4f12e514b53ba17c59c2c5fa75c as hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/f3ed23ac403772e0fe4ca9ee892544ec/C/1278f4f12e514b53ba17c59c2c5fa75c 2024-11-16T09:59:13,354 INFO [RS:0;037c58a19ddb:37267-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in f3ed23ac403772e0fe4ca9ee892544ec/C of f3ed23ac403772e0fe4ca9ee892544ec into 1278f4f12e514b53ba17c59c2c5fa75c(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-16T09:59:13,354 DEBUG [RS:0;037c58a19ddb:37267-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for f3ed23ac403772e0fe4ca9ee892544ec: 2024-11-16T09:59:13,354 INFO [RS:0;037c58a19ddb:37267-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731751137553.f3ed23ac403772e0fe4ca9ee892544ec., storeName=f3ed23ac403772e0fe4ca9ee892544ec/C, priority=13, startTime=1731751152905; duration=0sec 2024-11-16T09:59:13,354 DEBUG [RS:0;037c58a19ddb:37267-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-16T09:59:13,354 DEBUG [RS:0;037c58a19ddb:37267-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: f3ed23ac403772e0fe4ca9ee892544ec:C 2024-11-16T09:59:13,362 INFO [RS_FLUSH_OPERATIONS-regionserver/037c58a19ddb:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=51.44 KB at sequenceid=274 (bloomFilter=true), to=hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/f3ed23ac403772e0fe4ca9ee892544ec/.tmp/A/882d0e38ac3f4f40bbb1b64db18c4a2d 2024-11-16T09:59:13,369 DEBUG [RS_FLUSH_OPERATIONS-regionserver/037c58a19ddb:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/f3ed23ac403772e0fe4ca9ee892544ec/.tmp/B/79248da02305417c88dd121613236b79 is 50, key is test_row_0/B:col10/1731751152080/Put/seqid=0 2024-11-16T09:59:13,374 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37591 is added to blk_1073742134_1310 (size=12301) 2024-11-16T09:59:13,554 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3ed23ac403772e0fe4ca9ee892544ec, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T09:59:13,555 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] ipc.CallRunner(138): callId: 94 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34368 deadline: 1731751213553, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3ed23ac403772e0fe4ca9ee892544ec, server=037c58a19ddb,37267,1731751077844 2024-11-16T09:59:13,555 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731751137553.f3ed23ac403772e0fe4ca9ee892544ec., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 , the old value is region=TestAcidGuarantees,,1731751137553.f3ed23ac403772e0fe4ca9ee892544ec., hostname=037c58a19ddb,37267,1731751077844, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3ed23ac403772e0fe4ca9ee892544ec, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:59:13,555 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731751137553.f3ed23ac403772e0fe4ca9ee892544ec., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3ed23ac403772e0fe4ca9ee892544ec, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:59:13,555 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731751137553.f3ed23ac403772e0fe4ca9ee892544ec., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 because the exception is null or not the one we care about 2024-11-16T09:59:13,555 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3ed23ac403772e0fe4ca9ee892544ec, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T09:59:13,556 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] ipc.CallRunner(138): callId: 93 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34408 deadline: 1731751213553, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3ed23ac403772e0fe4ca9ee892544ec, server=037c58a19ddb,37267,1731751077844 2024-11-16T09:59:13,556 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3ed23ac403772e0fe4ca9ee892544ec, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T09:59:13,556 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731751137553.f3ed23ac403772e0fe4ca9ee892544ec., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 , the old value is region=TestAcidGuarantees,,1731751137553.f3ed23ac403772e0fe4ca9ee892544ec., hostname=037c58a19ddb,37267,1731751077844, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3ed23ac403772e0fe4ca9ee892544ec, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:59:13,556 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3ed23ac403772e0fe4ca9ee892544ec, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T09:59:13,556 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731751137553.f3ed23ac403772e0fe4ca9ee892544ec., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3ed23ac403772e0fe4ca9ee892544ec, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:59:13,556 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] ipc.CallRunner(138): callId: 89 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34414 deadline: 1731751213553, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3ed23ac403772e0fe4ca9ee892544ec, server=037c58a19ddb,37267,1731751077844 2024-11-16T09:59:13,556 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731751137553.f3ed23ac403772e0fe4ca9ee892544ec., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 because the exception is null or not the one we care about 2024-11-16T09:59:13,556 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] ipc.CallRunner(138): callId: 87 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34382 deadline: 1731751213553, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3ed23ac403772e0fe4ca9ee892544ec, server=037c58a19ddb,37267,1731751077844 2024-11-16T09:59:13,556 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731751137553.f3ed23ac403772e0fe4ca9ee892544ec., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 , the old value is region=TestAcidGuarantees,,1731751137553.f3ed23ac403772e0fe4ca9ee892544ec., hostname=037c58a19ddb,37267,1731751077844, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3ed23ac403772e0fe4ca9ee892544ec, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:59:13,556 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731751137553.f3ed23ac403772e0fe4ca9ee892544ec., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 , the old value is region=TestAcidGuarantees,,1731751137553.f3ed23ac403772e0fe4ca9ee892544ec., hostname=037c58a19ddb,37267,1731751077844, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3ed23ac403772e0fe4ca9ee892544ec, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:59:13,556 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3ed23ac403772e0fe4ca9ee892544ec, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T09:59:13,556 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731751137553.f3ed23ac403772e0fe4ca9ee892544ec., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3ed23ac403772e0fe4ca9ee892544ec, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:59:13,556 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731751137553.f3ed23ac403772e0fe4ca9ee892544ec., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3ed23ac403772e0fe4ca9ee892544ec, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:59:13,556 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731751137553.f3ed23ac403772e0fe4ca9ee892544ec., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 because the exception is null or not the one we care about 2024-11-16T09:59:13,556 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731751137553.f3ed23ac403772e0fe4ca9ee892544ec., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 because the exception is null or not the one we care about 2024-11-16T09:59:13,556 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] ipc.CallRunner(138): callId: 93 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34386 deadline: 1731751213553, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3ed23ac403772e0fe4ca9ee892544ec, server=037c58a19ddb,37267,1731751077844 2024-11-16T09:59:13,557 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731751137553.f3ed23ac403772e0fe4ca9ee892544ec., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 , the old value is region=TestAcidGuarantees,,1731751137553.f3ed23ac403772e0fe4ca9ee892544ec., hostname=037c58a19ddb,37267,1731751077844, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3ed23ac403772e0fe4ca9ee892544ec, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:59:13,557 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731751137553.f3ed23ac403772e0fe4ca9ee892544ec., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3ed23ac403772e0fe4ca9ee892544ec, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:59:13,557 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731751137553.f3ed23ac403772e0fe4ca9ee892544ec., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 because the exception is null or not the one we care about 2024-11-16T09:59:13,775 INFO [RS_FLUSH_OPERATIONS-regionserver/037c58a19ddb:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=51.44 KB at sequenceid=274 (bloomFilter=true), to=hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/f3ed23ac403772e0fe4ca9ee892544ec/.tmp/B/79248da02305417c88dd121613236b79 2024-11-16T09:59:13,783 DEBUG [RS_FLUSH_OPERATIONS-regionserver/037c58a19ddb:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/f3ed23ac403772e0fe4ca9ee892544ec/.tmp/C/04b64f8be1d64da9880fa2578af827ac is 50, key is test_row_0/C:col10/1731751152080/Put/seqid=0 2024-11-16T09:59:13,783 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37877 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=72 2024-11-16T09:59:13,787 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37591 is added to blk_1073742135_1311 (size=12301) 2024-11-16T09:59:13,788 INFO [RS_FLUSH_OPERATIONS-regionserver/037c58a19ddb:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=51.44 KB at sequenceid=274 (bloomFilter=true), to=hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/f3ed23ac403772e0fe4ca9ee892544ec/.tmp/C/04b64f8be1d64da9880fa2578af827ac 2024-11-16T09:59:13,792 DEBUG [RS_FLUSH_OPERATIONS-regionserver/037c58a19ddb:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/f3ed23ac403772e0fe4ca9ee892544ec/.tmp/A/882d0e38ac3f4f40bbb1b64db18c4a2d as hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/f3ed23ac403772e0fe4ca9ee892544ec/A/882d0e38ac3f4f40bbb1b64db18c4a2d 2024-11-16T09:59:13,796 INFO [RS_FLUSH_OPERATIONS-regionserver/037c58a19ddb:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/f3ed23ac403772e0fe4ca9ee892544ec/A/882d0e38ac3f4f40bbb1b64db18c4a2d, entries=150, sequenceid=274, filesize=12.0 K 2024-11-16T09:59:13,796 DEBUG [RS_FLUSH_OPERATIONS-regionserver/037c58a19ddb:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/f3ed23ac403772e0fe4ca9ee892544ec/.tmp/B/79248da02305417c88dd121613236b79 as hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/f3ed23ac403772e0fe4ca9ee892544ec/B/79248da02305417c88dd121613236b79 2024-11-16T09:59:13,800 INFO [RS_FLUSH_OPERATIONS-regionserver/037c58a19ddb:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/f3ed23ac403772e0fe4ca9ee892544ec/B/79248da02305417c88dd121613236b79, entries=150, sequenceid=274, filesize=12.0 K 2024-11-16T09:59:13,801 DEBUG [RS_FLUSH_OPERATIONS-regionserver/037c58a19ddb:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/f3ed23ac403772e0fe4ca9ee892544ec/.tmp/C/04b64f8be1d64da9880fa2578af827ac as hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/f3ed23ac403772e0fe4ca9ee892544ec/C/04b64f8be1d64da9880fa2578af827ac 2024-11-16T09:59:13,805 INFO [RS_FLUSH_OPERATIONS-regionserver/037c58a19ddb:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/f3ed23ac403772e0fe4ca9ee892544ec/C/04b64f8be1d64da9880fa2578af827ac, entries=150, sequenceid=274, filesize=12.0 K 2024-11-16T09:59:13,806 INFO [RS_FLUSH_OPERATIONS-regionserver/037c58a19ddb:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HRegion(3140): Finished flush of dataSize ~154.31 KB/158010, heapSize ~405 KB/414720, currentSize=46.96 KB/48090 for f3ed23ac403772e0fe4ca9ee892544ec in 853ms, sequenceid=274, compaction requested=false 2024-11-16T09:59:13,806 DEBUG [RS_FLUSH_OPERATIONS-regionserver/037c58a19ddb:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HRegion(2603): Flush status journal for f3ed23ac403772e0fe4ca9ee892544ec: 2024-11-16T09:59:13,807 DEBUG [RS_FLUSH_OPERATIONS-regionserver/037c58a19ddb:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731751137553.f3ed23ac403772e0fe4ca9ee892544ec. 2024-11-16T09:59:13,807 DEBUG [RS_FLUSH_OPERATIONS-regionserver/037c58a19ddb:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=73 2024-11-16T09:59:13,807 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37877 {}] master.HMaster(4169): Remote procedure done, pid=73 2024-11-16T09:59:13,810 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=73, resume processing ppid=72 2024-11-16T09:59:13,811 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=73, ppid=72, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.1610 sec 2024-11-16T09:59:13,812 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=72, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=72, table=TestAcidGuarantees in 1.1670 sec 2024-11-16T09:59:13,865 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] regionserver.HRegion(8855): Flush requested on f3ed23ac403772e0fe4ca9ee892544ec 2024-11-16T09:59:13,865 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-16T09:59:13,865 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing f3ed23ac403772e0fe4ca9ee892544ec 3/3 column families, dataSize=53.67 KB heapSize=141.38 KB 2024-11-16T09:59:13,871 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/f3ed23ac403772e0fe4ca9ee892544ec/.tmp/A/c2ff2a2750b14c829383d875cf110f93 is 50, key is test_row_0/A:col10/1731751153865/Put/seqid=0 2024-11-16T09:59:13,875 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37591 is added to blk_1073742136_1312 (size=12301) 2024-11-16T09:59:13,890 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3ed23ac403772e0fe4ca9ee892544ec, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T09:59:13,890 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] ipc.CallRunner(138): callId: 92 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34382 deadline: 1731751213885, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3ed23ac403772e0fe4ca9ee892544ec, server=037c58a19ddb,37267,1731751077844 2024-11-16T09:59:13,890 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3ed23ac403772e0fe4ca9ee892544ec, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T09:59:13,890 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] ipc.CallRunner(138): callId: 100 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34368 deadline: 1731751213885, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3ed23ac403772e0fe4ca9ee892544ec, server=037c58a19ddb,37267,1731751077844 2024-11-16T09:59:13,890 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731751137553.f3ed23ac403772e0fe4ca9ee892544ec., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 , the old value is region=TestAcidGuarantees,,1731751137553.f3ed23ac403772e0fe4ca9ee892544ec., hostname=037c58a19ddb,37267,1731751077844, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3ed23ac403772e0fe4ca9ee892544ec, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:59:13,890 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731751137553.f3ed23ac403772e0fe4ca9ee892544ec., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3ed23ac403772e0fe4ca9ee892544ec, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:59:13,890 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731751137553.f3ed23ac403772e0fe4ca9ee892544ec., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 because the exception is null or not the one we care about 2024-11-16T09:59:13,890 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731751137553.f3ed23ac403772e0fe4ca9ee892544ec., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 , the old value is region=TestAcidGuarantees,,1731751137553.f3ed23ac403772e0fe4ca9ee892544ec., hostname=037c58a19ddb,37267,1731751077844, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3ed23ac403772e0fe4ca9ee892544ec, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:59:13,890 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3ed23ac403772e0fe4ca9ee892544ec, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T09:59:13,890 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731751137553.f3ed23ac403772e0fe4ca9ee892544ec., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3ed23ac403772e0fe4ca9ee892544ec, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:59:13,890 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731751137553.f3ed23ac403772e0fe4ca9ee892544ec., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 because the exception is null or not the one we care about 2024-11-16T09:59:13,890 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] ipc.CallRunner(138): callId: 98 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34386 deadline: 1731751213886, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3ed23ac403772e0fe4ca9ee892544ec, server=037c58a19ddb,37267,1731751077844 2024-11-16T09:59:13,891 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731751137553.f3ed23ac403772e0fe4ca9ee892544ec., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 , the old value is region=TestAcidGuarantees,,1731751137553.f3ed23ac403772e0fe4ca9ee892544ec., hostname=037c58a19ddb,37267,1731751077844, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3ed23ac403772e0fe4ca9ee892544ec, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:59:13,891 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731751137553.f3ed23ac403772e0fe4ca9ee892544ec., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3ed23ac403772e0fe4ca9ee892544ec, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:59:13,891 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731751137553.f3ed23ac403772e0fe4ca9ee892544ec., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 because the exception is null or not the one we care about 2024-11-16T09:59:13,892 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3ed23ac403772e0fe4ca9ee892544ec, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T09:59:13,892 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] ipc.CallRunner(138): callId: 99 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34408 deadline: 1731751213888, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3ed23ac403772e0fe4ca9ee892544ec, server=037c58a19ddb,37267,1731751077844 2024-11-16T09:59:13,892 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731751137553.f3ed23ac403772e0fe4ca9ee892544ec., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 , the old value is region=TestAcidGuarantees,,1731751137553.f3ed23ac403772e0fe4ca9ee892544ec., hostname=037c58a19ddb,37267,1731751077844, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3ed23ac403772e0fe4ca9ee892544ec, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:59:13,892 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731751137553.f3ed23ac403772e0fe4ca9ee892544ec., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3ed23ac403772e0fe4ca9ee892544ec, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:59:13,892 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731751137553.f3ed23ac403772e0fe4ca9ee892544ec., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 because the exception is null or not the one we care about 2024-11-16T09:59:13,893 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3ed23ac403772e0fe4ca9ee892544ec, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T09:59:13,893 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] ipc.CallRunner(138): callId: 95 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34414 deadline: 1731751213890, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3ed23ac403772e0fe4ca9ee892544ec, server=037c58a19ddb,37267,1731751077844 2024-11-16T09:59:13,894 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731751137553.f3ed23ac403772e0fe4ca9ee892544ec., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 , the old value is region=TestAcidGuarantees,,1731751137553.f3ed23ac403772e0fe4ca9ee892544ec., hostname=037c58a19ddb,37267,1731751077844, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3ed23ac403772e0fe4ca9ee892544ec, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:59:13,894 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731751137553.f3ed23ac403772e0fe4ca9ee892544ec., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3ed23ac403772e0fe4ca9ee892544ec, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:59:13,894 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731751137553.f3ed23ac403772e0fe4ca9ee892544ec., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 because the exception is null or not the one we care about 2024-11-16T09:59:13,994 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3ed23ac403772e0fe4ca9ee892544ec, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T09:59:13,994 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] ipc.CallRunner(138): callId: 93 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34382 deadline: 1731751213993, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3ed23ac403772e0fe4ca9ee892544ec, server=037c58a19ddb,37267,1731751077844 2024-11-16T09:59:13,994 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731751137553.f3ed23ac403772e0fe4ca9ee892544ec., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 , the old value is region=TestAcidGuarantees,,1731751137553.f3ed23ac403772e0fe4ca9ee892544ec., hostname=037c58a19ddb,37267,1731751077844, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3ed23ac403772e0fe4ca9ee892544ec, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:59:13,994 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3ed23ac403772e0fe4ca9ee892544ec, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T09:59:13,994 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731751137553.f3ed23ac403772e0fe4ca9ee892544ec., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3ed23ac403772e0fe4ca9ee892544ec, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:59:13,994 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731751137553.f3ed23ac403772e0fe4ca9ee892544ec., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 because the exception is null or not the one we care about 2024-11-16T09:59:13,994 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] ipc.CallRunner(138): callId: 101 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34368 deadline: 1731751213993, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3ed23ac403772e0fe4ca9ee892544ec, server=037c58a19ddb,37267,1731751077844 2024-11-16T09:59:13,994 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3ed23ac403772e0fe4ca9ee892544ec, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T09:59:13,995 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] ipc.CallRunner(138): callId: 99 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34386 deadline: 1731751213993, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3ed23ac403772e0fe4ca9ee892544ec, server=037c58a19ddb,37267,1731751077844 2024-11-16T09:59:13,995 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731751137553.f3ed23ac403772e0fe4ca9ee892544ec., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 , the old value is region=TestAcidGuarantees,,1731751137553.f3ed23ac403772e0fe4ca9ee892544ec., hostname=037c58a19ddb,37267,1731751077844, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3ed23ac403772e0fe4ca9ee892544ec, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:59:13,995 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731751137553.f3ed23ac403772e0fe4ca9ee892544ec., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3ed23ac403772e0fe4ca9ee892544ec, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:59:13,995 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731751137553.f3ed23ac403772e0fe4ca9ee892544ec., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 because the exception is null or not the one we care about 2024-11-16T09:59:13,995 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731751137553.f3ed23ac403772e0fe4ca9ee892544ec., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 , the old value is region=TestAcidGuarantees,,1731751137553.f3ed23ac403772e0fe4ca9ee892544ec., hostname=037c58a19ddb,37267,1731751077844, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3ed23ac403772e0fe4ca9ee892544ec, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:59:13,995 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731751137553.f3ed23ac403772e0fe4ca9ee892544ec., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3ed23ac403772e0fe4ca9ee892544ec, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:59:13,995 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731751137553.f3ed23ac403772e0fe4ca9ee892544ec., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 because the exception is null or not the one we care about 2024-11-16T09:59:14,005 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3ed23ac403772e0fe4ca9ee892544ec, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T09:59:14,006 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] ipc.CallRunner(138): callId: 100 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34408 deadline: 1731751214003, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3ed23ac403772e0fe4ca9ee892544ec, server=037c58a19ddb,37267,1731751077844 2024-11-16T09:59:14,006 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3ed23ac403772e0fe4ca9ee892544ec, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T09:59:14,006 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731751137553.f3ed23ac403772e0fe4ca9ee892544ec., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 , the old value is region=TestAcidGuarantees,,1731751137553.f3ed23ac403772e0fe4ca9ee892544ec., hostname=037c58a19ddb,37267,1731751077844, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3ed23ac403772e0fe4ca9ee892544ec, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:59:14,006 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] ipc.CallRunner(138): callId: 96 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34414 deadline: 1731751214003, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3ed23ac403772e0fe4ca9ee892544ec, server=037c58a19ddb,37267,1731751077844 2024-11-16T09:59:14,006 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731751137553.f3ed23ac403772e0fe4ca9ee892544ec., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3ed23ac403772e0fe4ca9ee892544ec, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:59:14,006 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731751137553.f3ed23ac403772e0fe4ca9ee892544ec., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 because the exception is null or not the one we care about 2024-11-16T09:59:14,006 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731751137553.f3ed23ac403772e0fe4ca9ee892544ec., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 , the old value is region=TestAcidGuarantees,,1731751137553.f3ed23ac403772e0fe4ca9ee892544ec., hostname=037c58a19ddb,37267,1731751077844, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3ed23ac403772e0fe4ca9ee892544ec, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:59:14,006 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731751137553.f3ed23ac403772e0fe4ca9ee892544ec., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3ed23ac403772e0fe4ca9ee892544ec, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:59:14,006 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731751137553.f3ed23ac403772e0fe4ca9ee892544ec., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 because the exception is null or not the one we care about 2024-11-16T09:59:14,205 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3ed23ac403772e0fe4ca9ee892544ec, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T09:59:14,205 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] ipc.CallRunner(138): callId: 94 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34382 deadline: 1731751214203, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3ed23ac403772e0fe4ca9ee892544ec, server=037c58a19ddb,37267,1731751077844 2024-11-16T09:59:14,206 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3ed23ac403772e0fe4ca9ee892544ec, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T09:59:14,206 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731751137553.f3ed23ac403772e0fe4ca9ee892544ec., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 , the old value is region=TestAcidGuarantees,,1731751137553.f3ed23ac403772e0fe4ca9ee892544ec., hostname=037c58a19ddb,37267,1731751077844, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3ed23ac403772e0fe4ca9ee892544ec, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:59:14,206 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] ipc.CallRunner(138): callId: 102 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34368 deadline: 1731751214203, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3ed23ac403772e0fe4ca9ee892544ec, server=037c58a19ddb,37267,1731751077844 2024-11-16T09:59:14,206 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731751137553.f3ed23ac403772e0fe4ca9ee892544ec., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3ed23ac403772e0fe4ca9ee892544ec, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:59:14,206 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731751137553.f3ed23ac403772e0fe4ca9ee892544ec., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 because the exception is null or not the one we care about 2024-11-16T09:59:14,206 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3ed23ac403772e0fe4ca9ee892544ec, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T09:59:14,206 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] ipc.CallRunner(138): callId: 100 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34386 deadline: 1731751214203, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3ed23ac403772e0fe4ca9ee892544ec, server=037c58a19ddb,37267,1731751077844 2024-11-16T09:59:14,206 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731751137553.f3ed23ac403772e0fe4ca9ee892544ec., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 , the old value is region=TestAcidGuarantees,,1731751137553.f3ed23ac403772e0fe4ca9ee892544ec., hostname=037c58a19ddb,37267,1731751077844, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3ed23ac403772e0fe4ca9ee892544ec, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:59:14,206 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731751137553.f3ed23ac403772e0fe4ca9ee892544ec., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3ed23ac403772e0fe4ca9ee892544ec, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:59:14,206 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731751137553.f3ed23ac403772e0fe4ca9ee892544ec., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 because the exception is null or not the one we care about 2024-11-16T09:59:14,206 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731751137553.f3ed23ac403772e0fe4ca9ee892544ec., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 , the old value is region=TestAcidGuarantees,,1731751137553.f3ed23ac403772e0fe4ca9ee892544ec., hostname=037c58a19ddb,37267,1731751077844, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3ed23ac403772e0fe4ca9ee892544ec, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:59:14,206 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731751137553.f3ed23ac403772e0fe4ca9ee892544ec., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3ed23ac403772e0fe4ca9ee892544ec, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:59:14,206 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731751137553.f3ed23ac403772e0fe4ca9ee892544ec., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 because the exception is null or not the one we care about 2024-11-16T09:59:14,214 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3ed23ac403772e0fe4ca9ee892544ec, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T09:59:14,214 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] ipc.CallRunner(138): callId: 101 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34408 deadline: 1731751214213, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3ed23ac403772e0fe4ca9ee892544ec, server=037c58a19ddb,37267,1731751077844 2024-11-16T09:59:14,215 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3ed23ac403772e0fe4ca9ee892544ec, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T09:59:14,215 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731751137553.f3ed23ac403772e0fe4ca9ee892544ec., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 , the old value is region=TestAcidGuarantees,,1731751137553.f3ed23ac403772e0fe4ca9ee892544ec., hostname=037c58a19ddb,37267,1731751077844, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3ed23ac403772e0fe4ca9ee892544ec, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:59:14,215 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] ipc.CallRunner(138): callId: 97 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34414 deadline: 1731751214213, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3ed23ac403772e0fe4ca9ee892544ec, server=037c58a19ddb,37267,1731751077844 2024-11-16T09:59:14,215 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731751137553.f3ed23ac403772e0fe4ca9ee892544ec., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3ed23ac403772e0fe4ca9ee892544ec, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:59:14,215 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731751137553.f3ed23ac403772e0fe4ca9ee892544ec., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 because the exception is null or not the one we care about 2024-11-16T09:59:14,215 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731751137553.f3ed23ac403772e0fe4ca9ee892544ec., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 , the old value is region=TestAcidGuarantees,,1731751137553.f3ed23ac403772e0fe4ca9ee892544ec., hostname=037c58a19ddb,37267,1731751077844, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3ed23ac403772e0fe4ca9ee892544ec, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:59:14,215 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731751137553.f3ed23ac403772e0fe4ca9ee892544ec., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3ed23ac403772e0fe4ca9ee892544ec, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:59:14,215 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731751137553.f3ed23ac403772e0fe4ca9ee892544ec., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 because the exception is null or not the one we care about 2024-11-16T09:59:14,276 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=290 (bloomFilter=true), to=hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/f3ed23ac403772e0fe4ca9ee892544ec/.tmp/A/c2ff2a2750b14c829383d875cf110f93 2024-11-16T09:59:14,284 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/f3ed23ac403772e0fe4ca9ee892544ec/.tmp/B/2dff7f545b324a0c9eb39bd60f22cc2a is 50, key is test_row_0/B:col10/1731751153865/Put/seqid=0 2024-11-16T09:59:14,287 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37591 is added to blk_1073742137_1313 (size=12301) 2024-11-16T09:59:14,515 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3ed23ac403772e0fe4ca9ee892544ec, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T09:59:14,515 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] ipc.CallRunner(138): callId: 95 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34382 deadline: 1731751214513, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3ed23ac403772e0fe4ca9ee892544ec, server=037c58a19ddb,37267,1731751077844 2024-11-16T09:59:14,515 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3ed23ac403772e0fe4ca9ee892544ec, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T09:59:14,516 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] ipc.CallRunner(138): callId: 103 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34368 deadline: 1731751214513, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3ed23ac403772e0fe4ca9ee892544ec, server=037c58a19ddb,37267,1731751077844 2024-11-16T09:59:14,516 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731751137553.f3ed23ac403772e0fe4ca9ee892544ec., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 , the old value is region=TestAcidGuarantees,,1731751137553.f3ed23ac403772e0fe4ca9ee892544ec., hostname=037c58a19ddb,37267,1731751077844, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3ed23ac403772e0fe4ca9ee892544ec, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:59:14,516 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731751137553.f3ed23ac403772e0fe4ca9ee892544ec., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3ed23ac403772e0fe4ca9ee892544ec, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:59:14,516 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731751137553.f3ed23ac403772e0fe4ca9ee892544ec., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 because the exception is null or not the one we care about 2024-11-16T09:59:14,516 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3ed23ac403772e0fe4ca9ee892544ec, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T09:59:14,516 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731751137553.f3ed23ac403772e0fe4ca9ee892544ec., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 , the old value is region=TestAcidGuarantees,,1731751137553.f3ed23ac403772e0fe4ca9ee892544ec., hostname=037c58a19ddb,37267,1731751077844, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3ed23ac403772e0fe4ca9ee892544ec, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:59:14,516 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731751137553.f3ed23ac403772e0fe4ca9ee892544ec., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3ed23ac403772e0fe4ca9ee892544ec, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:59:14,516 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] ipc.CallRunner(138): callId: 101 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34386 deadline: 1731751214513, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3ed23ac403772e0fe4ca9ee892544ec, server=037c58a19ddb,37267,1731751077844 2024-11-16T09:59:14,516 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731751137553.f3ed23ac403772e0fe4ca9ee892544ec., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 because the exception is null or not the one we care about 2024-11-16T09:59:14,516 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731751137553.f3ed23ac403772e0fe4ca9ee892544ec., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 , the old value is region=TestAcidGuarantees,,1731751137553.f3ed23ac403772e0fe4ca9ee892544ec., hostname=037c58a19ddb,37267,1731751077844, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3ed23ac403772e0fe4ca9ee892544ec, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:59:14,516 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731751137553.f3ed23ac403772e0fe4ca9ee892544ec., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3ed23ac403772e0fe4ca9ee892544ec, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:59:14,516 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731751137553.f3ed23ac403772e0fe4ca9ee892544ec., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 because the exception is null or not the one we care about 2024-11-16T09:59:14,524 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3ed23ac403772e0fe4ca9ee892544ec, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T09:59:14,524 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] ipc.CallRunner(138): callId: 102 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34408 deadline: 1731751214523, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3ed23ac403772e0fe4ca9ee892544ec, server=037c58a19ddb,37267,1731751077844 2024-11-16T09:59:14,524 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731751137553.f3ed23ac403772e0fe4ca9ee892544ec., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 , the old value is region=TestAcidGuarantees,,1731751137553.f3ed23ac403772e0fe4ca9ee892544ec., hostname=037c58a19ddb,37267,1731751077844, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3ed23ac403772e0fe4ca9ee892544ec, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:59:14,524 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3ed23ac403772e0fe4ca9ee892544ec, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T09:59:14,525 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731751137553.f3ed23ac403772e0fe4ca9ee892544ec., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3ed23ac403772e0fe4ca9ee892544ec, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:59:14,525 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731751137553.f3ed23ac403772e0fe4ca9ee892544ec., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 because the exception is null or not the one we care about 2024-11-16T09:59:14,525 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] ipc.CallRunner(138): callId: 98 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34414 deadline: 1731751214523, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3ed23ac403772e0fe4ca9ee892544ec, server=037c58a19ddb,37267,1731751077844 2024-11-16T09:59:14,525 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731751137553.f3ed23ac403772e0fe4ca9ee892544ec., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 , the old value is region=TestAcidGuarantees,,1731751137553.f3ed23ac403772e0fe4ca9ee892544ec., hostname=037c58a19ddb,37267,1731751077844, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3ed23ac403772e0fe4ca9ee892544ec, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:59:14,525 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731751137553.f3ed23ac403772e0fe4ca9ee892544ec., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3ed23ac403772e0fe4ca9ee892544ec, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:59:14,525 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731751137553.f3ed23ac403772e0fe4ca9ee892544ec., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 because the exception is null or not the one we care about 2024-11-16T09:59:14,688 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=290 (bloomFilter=true), to=hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/f3ed23ac403772e0fe4ca9ee892544ec/.tmp/B/2dff7f545b324a0c9eb39bd60f22cc2a 2024-11-16T09:59:14,696 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/f3ed23ac403772e0fe4ca9ee892544ec/.tmp/C/246486ae295b4ce6b3d0df8dfcebfe51 is 50, key is test_row_0/C:col10/1731751153865/Put/seqid=0 2024-11-16T09:59:14,700 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37591 is added to blk_1073742138_1314 (size=12301) 2024-11-16T09:59:14,802 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37877 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=72 2024-11-16T09:59:14,803 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-16T09:59:14,803 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37877 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-16T09:59:14,804 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37877 {}] procedure2.ProcedureExecutor(1139): Stored pid=74, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=74, table=TestAcidGuarantees 2024-11-16T09:59:14,805 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37877 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=74 2024-11-16T09:59:14,805 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=74, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=74, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-16T09:59:14,806 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=74, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=74, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-16T09:59:14,806 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=75, ppid=74, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-16T09:59:14,913 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37877 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=74 2024-11-16T09:59:14,958 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=37267 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=75 2024-11-16T09:59:14,958 DEBUG [RS_FLUSH_OPERATIONS-regionserver/037c58a19ddb:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731751137553.f3ed23ac403772e0fe4ca9ee892544ec. 2024-11-16T09:59:14,958 DEBUG [RS_FLUSH_OPERATIONS-regionserver/037c58a19ddb:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731751137553.f3ed23ac403772e0fe4ca9ee892544ec. as already flushing 2024-11-16T09:59:14,958 DEBUG [RS_FLUSH_OPERATIONS-regionserver/037c58a19ddb:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731751137553.f3ed23ac403772e0fe4ca9ee892544ec. 2024-11-16T09:59:14,959 ERROR [RS_FLUSH_OPERATIONS-regionserver/037c58a19ddb:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] handler.RSProcedureHandler(58): pid=75 java.io.IOException: Unable to complete flush {ENCODED => f3ed23ac403772e0fe4ca9ee892544ec, NAME => 'TestAcidGuarantees,,1731751137553.f3ed23ac403772e0fe4ca9ee892544ec.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T09:59:14,959 DEBUG [RS_FLUSH_OPERATIONS-regionserver/037c58a19ddb:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=75 java.io.IOException: Unable to complete flush {ENCODED => f3ed23ac403772e0fe4ca9ee892544ec, NAME => 'TestAcidGuarantees,,1731751137553.f3ed23ac403772e0fe4ca9ee892544ec.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T09:59:14,959 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37877 {}] master.HMaster(4177): Remote procedure failed, pid=75 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => f3ed23ac403772e0fe4ca9ee892544ec, NAME => 'TestAcidGuarantees,,1731751137553.f3ed23ac403772e0fe4ca9ee892544ec.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => f3ed23ac403772e0fe4ca9ee892544ec, NAME => 'TestAcidGuarantees,,1731751137553.f3ed23ac403772e0fe4ca9ee892544ec.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T09:59:15,025 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3ed23ac403772e0fe4ca9ee892544ec, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T09:59:15,026 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] ipc.CallRunner(138): callId: 96 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34382 deadline: 1731751215023, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3ed23ac403772e0fe4ca9ee892544ec, server=037c58a19ddb,37267,1731751077844 2024-11-16T09:59:15,026 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731751137553.f3ed23ac403772e0fe4ca9ee892544ec., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 , the old value is region=TestAcidGuarantees,,1731751137553.f3ed23ac403772e0fe4ca9ee892544ec., hostname=037c58a19ddb,37267,1731751077844, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3ed23ac403772e0fe4ca9ee892544ec, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:59:15,026 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3ed23ac403772e0fe4ca9ee892544ec, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T09:59:15,026 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731751137553.f3ed23ac403772e0fe4ca9ee892544ec., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3ed23ac403772e0fe4ca9ee892544ec, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:59:15,026 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731751137553.f3ed23ac403772e0fe4ca9ee892544ec., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 because the exception is null or not the one we care about 2024-11-16T09:59:15,026 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] ipc.CallRunner(138): callId: 104 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34368 deadline: 1731751215023, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3ed23ac403772e0fe4ca9ee892544ec, server=037c58a19ddb,37267,1731751077844 2024-11-16T09:59:15,026 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3ed23ac403772e0fe4ca9ee892544ec, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T09:59:15,026 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731751137553.f3ed23ac403772e0fe4ca9ee892544ec., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 , the old value is region=TestAcidGuarantees,,1731751137553.f3ed23ac403772e0fe4ca9ee892544ec., hostname=037c58a19ddb,37267,1731751077844, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3ed23ac403772e0fe4ca9ee892544ec, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:59:15,026 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731751137553.f3ed23ac403772e0fe4ca9ee892544ec., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3ed23ac403772e0fe4ca9ee892544ec, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:59:15,026 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731751137553.f3ed23ac403772e0fe4ca9ee892544ec., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 because the exception is null or not the one we care about 2024-11-16T09:59:15,026 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] ipc.CallRunner(138): callId: 102 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34386 deadline: 1731751215023, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3ed23ac403772e0fe4ca9ee892544ec, server=037c58a19ddb,37267,1731751077844 2024-11-16T09:59:15,026 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731751137553.f3ed23ac403772e0fe4ca9ee892544ec., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 , the old value is region=TestAcidGuarantees,,1731751137553.f3ed23ac403772e0fe4ca9ee892544ec., hostname=037c58a19ddb,37267,1731751077844, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3ed23ac403772e0fe4ca9ee892544ec, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:59:15,026 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731751137553.f3ed23ac403772e0fe4ca9ee892544ec., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3ed23ac403772e0fe4ca9ee892544ec, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:59:15,026 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731751137553.f3ed23ac403772e0fe4ca9ee892544ec., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 because the exception is null or not the one we care about 2024-11-16T09:59:15,035 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3ed23ac403772e0fe4ca9ee892544ec, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T09:59:15,035 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] ipc.CallRunner(138): callId: 103 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34408 deadline: 1731751215033, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3ed23ac403772e0fe4ca9ee892544ec, server=037c58a19ddb,37267,1731751077844 2024-11-16T09:59:15,035 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731751137553.f3ed23ac403772e0fe4ca9ee892544ec., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 , the old value is region=TestAcidGuarantees,,1731751137553.f3ed23ac403772e0fe4ca9ee892544ec., hostname=037c58a19ddb,37267,1731751077844, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3ed23ac403772e0fe4ca9ee892544ec, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:59:15,035 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731751137553.f3ed23ac403772e0fe4ca9ee892544ec., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3ed23ac403772e0fe4ca9ee892544ec, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:59:15,035 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731751137553.f3ed23ac403772e0fe4ca9ee892544ec., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 because the exception is null or not the one we care about 2024-11-16T09:59:15,035 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3ed23ac403772e0fe4ca9ee892544ec, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T09:59:15,035 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] ipc.CallRunner(138): callId: 99 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34414 deadline: 1731751215033, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3ed23ac403772e0fe4ca9ee892544ec, server=037c58a19ddb,37267,1731751077844 2024-11-16T09:59:15,035 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731751137553.f3ed23ac403772e0fe4ca9ee892544ec., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 , the old value is region=TestAcidGuarantees,,1731751137553.f3ed23ac403772e0fe4ca9ee892544ec., hostname=037c58a19ddb,37267,1731751077844, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3ed23ac403772e0fe4ca9ee892544ec, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:59:15,035 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731751137553.f3ed23ac403772e0fe4ca9ee892544ec., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3ed23ac403772e0fe4ca9ee892544ec, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:59:15,035 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731751137553.f3ed23ac403772e0fe4ca9ee892544ec., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 because the exception is null or not the one we care about 2024-11-16T09:59:15,101 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=290 (bloomFilter=true), to=hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/f3ed23ac403772e0fe4ca9ee892544ec/.tmp/C/246486ae295b4ce6b3d0df8dfcebfe51 2024-11-16T09:59:15,106 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/f3ed23ac403772e0fe4ca9ee892544ec/.tmp/A/c2ff2a2750b14c829383d875cf110f93 as hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/f3ed23ac403772e0fe4ca9ee892544ec/A/c2ff2a2750b14c829383d875cf110f93 2024-11-16T09:59:15,110 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/f3ed23ac403772e0fe4ca9ee892544ec/A/c2ff2a2750b14c829383d875cf110f93, entries=150, sequenceid=290, filesize=12.0 K 2024-11-16T09:59:15,110 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/f3ed23ac403772e0fe4ca9ee892544ec/.tmp/B/2dff7f545b324a0c9eb39bd60f22cc2a as hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/f3ed23ac403772e0fe4ca9ee892544ec/B/2dff7f545b324a0c9eb39bd60f22cc2a 2024-11-16T09:59:15,111 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=37267 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=75 2024-11-16T09:59:15,111 DEBUG [RS_FLUSH_OPERATIONS-regionserver/037c58a19ddb:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731751137553.f3ed23ac403772e0fe4ca9ee892544ec. 2024-11-16T09:59:15,112 DEBUG [RS_FLUSH_OPERATIONS-regionserver/037c58a19ddb:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731751137553.f3ed23ac403772e0fe4ca9ee892544ec. as already flushing 2024-11-16T09:59:15,112 DEBUG [RS_FLUSH_OPERATIONS-regionserver/037c58a19ddb:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731751137553.f3ed23ac403772e0fe4ca9ee892544ec. 2024-11-16T09:59:15,112 ERROR [RS_FLUSH_OPERATIONS-regionserver/037c58a19ddb:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] handler.RSProcedureHandler(58): pid=75 java.io.IOException: Unable to complete flush {ENCODED => f3ed23ac403772e0fe4ca9ee892544ec, NAME => 'TestAcidGuarantees,,1731751137553.f3ed23ac403772e0fe4ca9ee892544ec.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T09:59:15,112 DEBUG [RS_FLUSH_OPERATIONS-regionserver/037c58a19ddb:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=75 java.io.IOException: Unable to complete flush {ENCODED => f3ed23ac403772e0fe4ca9ee892544ec, NAME => 'TestAcidGuarantees,,1731751137553.f3ed23ac403772e0fe4ca9ee892544ec.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T09:59:15,112 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37877 {}] master.HMaster(4177): Remote procedure failed, pid=75 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => f3ed23ac403772e0fe4ca9ee892544ec, NAME => 'TestAcidGuarantees,,1731751137553.f3ed23ac403772e0fe4ca9ee892544ec.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => f3ed23ac403772e0fe4ca9ee892544ec, NAME => 'TestAcidGuarantees,,1731751137553.f3ed23ac403772e0fe4ca9ee892544ec.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T09:59:15,115 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/f3ed23ac403772e0fe4ca9ee892544ec/B/2dff7f545b324a0c9eb39bd60f22cc2a, entries=150, sequenceid=290, filesize=12.0 K 2024-11-16T09:59:15,116 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/f3ed23ac403772e0fe4ca9ee892544ec/.tmp/C/246486ae295b4ce6b3d0df8dfcebfe51 as hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/f3ed23ac403772e0fe4ca9ee892544ec/C/246486ae295b4ce6b3d0df8dfcebfe51 2024-11-16T09:59:15,122 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37877 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=74 2024-11-16T09:59:15,126 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/f3ed23ac403772e0fe4ca9ee892544ec/C/246486ae295b4ce6b3d0df8dfcebfe51, entries=150, sequenceid=290, filesize=12.0 K 2024-11-16T09:59:15,127 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~67.09 KB/68700, heapSize ~176.48 KB/180720, currentSize=134.18 KB/137400 for f3ed23ac403772e0fe4ca9ee892544ec in 1262ms, sequenceid=290, compaction requested=true 2024-11-16T09:59:15,128 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for f3ed23ac403772e0fe4ca9ee892544ec: 2024-11-16T09:59:15,128 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store f3ed23ac403772e0fe4ca9ee892544ec:A, priority=-2147483648, current under compaction store size is 1 2024-11-16T09:59:15,128 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-16T09:59:15,128 DEBUG [RS:0;037c58a19ddb:37267-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-16T09:59:15,128 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store f3ed23ac403772e0fe4ca9ee892544ec:B, priority=-2147483648, current under compaction store size is 2 2024-11-16T09:59:15,128 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-16T09:59:15,128 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store f3ed23ac403772e0fe4ca9ee892544ec:C, priority=-2147483648, current under compaction store size is 3 2024-11-16T09:59:15,128 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-16T09:59:15,128 DEBUG [RS:0;037c58a19ddb:37267-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-16T09:59:15,129 DEBUG [RS:0;037c58a19ddb:37267-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37333 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-16T09:59:15,129 DEBUG [RS:0;037c58a19ddb:37267-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37333 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-16T09:59:15,129 DEBUG [RS:0;037c58a19ddb:37267-longCompactions-0 {}] regionserver.HStore(1541): f3ed23ac403772e0fe4ca9ee892544ec/B is initiating minor compaction (all files) 2024-11-16T09:59:15,129 DEBUG [RS:0;037c58a19ddb:37267-shortCompactions-0 {}] regionserver.HStore(1541): f3ed23ac403772e0fe4ca9ee892544ec/A is initiating minor compaction (all files) 2024-11-16T09:59:15,129 INFO [RS:0;037c58a19ddb:37267-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of f3ed23ac403772e0fe4ca9ee892544ec/A in TestAcidGuarantees,,1731751137553.f3ed23ac403772e0fe4ca9ee892544ec. 2024-11-16T09:59:15,129 INFO [RS:0;037c58a19ddb:37267-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of f3ed23ac403772e0fe4ca9ee892544ec/B in TestAcidGuarantees,,1731751137553.f3ed23ac403772e0fe4ca9ee892544ec. 2024-11-16T09:59:15,129 INFO [RS:0;037c58a19ddb:37267-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/f3ed23ac403772e0fe4ca9ee892544ec/A/1775c312f3b740ccb57738318487b1ee, hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/f3ed23ac403772e0fe4ca9ee892544ec/A/882d0e38ac3f4f40bbb1b64db18c4a2d, hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/f3ed23ac403772e0fe4ca9ee892544ec/A/c2ff2a2750b14c829383d875cf110f93] into tmpdir=hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/f3ed23ac403772e0fe4ca9ee892544ec/.tmp, totalSize=36.5 K 2024-11-16T09:59:15,129 INFO [RS:0;037c58a19ddb:37267-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/f3ed23ac403772e0fe4ca9ee892544ec/B/3df0f2796e9848a6af1729c89fb64a31, hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/f3ed23ac403772e0fe4ca9ee892544ec/B/79248da02305417c88dd121613236b79, hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/f3ed23ac403772e0fe4ca9ee892544ec/B/2dff7f545b324a0c9eb39bd60f22cc2a] into tmpdir=hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/f3ed23ac403772e0fe4ca9ee892544ec/.tmp, totalSize=36.5 K 2024-11-16T09:59:15,129 DEBUG [RS:0;037c58a19ddb:37267-shortCompactions-0 {}] compactions.Compactor(225): Compacting 1775c312f3b740ccb57738318487b1ee, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=248, earliestPutTs=1731751152043 2024-11-16T09:59:15,130 DEBUG [RS:0;037c58a19ddb:37267-longCompactions-0 {}] compactions.Compactor(225): Compacting 3df0f2796e9848a6af1729c89fb64a31, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=248, earliestPutTs=1731751152043 2024-11-16T09:59:15,130 DEBUG [RS:0;037c58a19ddb:37267-shortCompactions-0 {}] compactions.Compactor(225): Compacting 882d0e38ac3f4f40bbb1b64db18c4a2d, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=274, earliestPutTs=1731751152070 2024-11-16T09:59:15,130 DEBUG [RS:0;037c58a19ddb:37267-longCompactions-0 {}] compactions.Compactor(225): Compacting 79248da02305417c88dd121613236b79, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=274, earliestPutTs=1731751152070 2024-11-16T09:59:15,130 DEBUG [RS:0;037c58a19ddb:37267-shortCompactions-0 {}] compactions.Compactor(225): Compacting c2ff2a2750b14c829383d875cf110f93, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=290, earliestPutTs=1731751153227 2024-11-16T09:59:15,131 DEBUG [RS:0;037c58a19ddb:37267-longCompactions-0 {}] compactions.Compactor(225): Compacting 2dff7f545b324a0c9eb39bd60f22cc2a, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=290, earliestPutTs=1731751153227 2024-11-16T09:59:15,142 INFO [RS:0;037c58a19ddb:37267-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): f3ed23ac403772e0fe4ca9ee892544ec#A#compaction#270 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-16T09:59:15,142 INFO [RS:0;037c58a19ddb:37267-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): f3ed23ac403772e0fe4ca9ee892544ec#B#compaction#271 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-16T09:59:15,143 DEBUG [RS:0;037c58a19ddb:37267-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/f3ed23ac403772e0fe4ca9ee892544ec/.tmp/A/12aace3b248347fc94ca8f8143923c21 is 50, key is test_row_0/A:col10/1731751153865/Put/seqid=0 2024-11-16T09:59:15,143 DEBUG [RS:0;037c58a19ddb:37267-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/f3ed23ac403772e0fe4ca9ee892544ec/.tmp/B/5d60f3c9b044415cbf7981c5183581ba is 50, key is test_row_0/B:col10/1731751153865/Put/seqid=0 2024-11-16T09:59:15,146 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37591 is added to blk_1073742139_1315 (size=12983) 2024-11-16T09:59:15,147 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37591 is added to blk_1073742140_1316 (size=12983) 2024-11-16T09:59:15,265 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=37267 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=75 2024-11-16T09:59:15,265 DEBUG [RS_FLUSH_OPERATIONS-regionserver/037c58a19ddb:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731751137553.f3ed23ac403772e0fe4ca9ee892544ec. 2024-11-16T09:59:15,265 INFO [RS_FLUSH_OPERATIONS-regionserver/037c58a19ddb:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HRegion(2902): Flushing f3ed23ac403772e0fe4ca9ee892544ec 3/3 column families, dataSize=134.18 KB heapSize=352.31 KB 2024-11-16T09:59:15,270 DEBUG [RS_FLUSH_OPERATIONS-regionserver/037c58a19ddb:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/f3ed23ac403772e0fe4ca9ee892544ec/.tmp/A/b8255e158316461c9e6faf3a1fcf57f1 is 50, key is test_row_0/A:col10/1731751153887/Put/seqid=0 2024-11-16T09:59:15,273 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37591 is added to blk_1073742141_1317 (size=12301) 2024-11-16T09:59:15,275 INFO [RS_FLUSH_OPERATIONS-regionserver/037c58a19ddb:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=313 (bloomFilter=true), to=hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/f3ed23ac403772e0fe4ca9ee892544ec/.tmp/A/b8255e158316461c9e6faf3a1fcf57f1 2024-11-16T09:59:15,283 DEBUG [RS_FLUSH_OPERATIONS-regionserver/037c58a19ddb:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/f3ed23ac403772e0fe4ca9ee892544ec/.tmp/B/9c2850d44afa4fc7baca1f5aa115bb38 is 50, key is test_row_0/B:col10/1731751153887/Put/seqid=0 2024-11-16T09:59:15,286 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37591 is added to blk_1073742142_1318 (size=12301) 2024-11-16T09:59:15,433 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37877 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=74 2024-11-16T09:59:15,553 DEBUG [RS:0;037c58a19ddb:37267-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/f3ed23ac403772e0fe4ca9ee892544ec/.tmp/B/5d60f3c9b044415cbf7981c5183581ba as hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/f3ed23ac403772e0fe4ca9ee892544ec/B/5d60f3c9b044415cbf7981c5183581ba 2024-11-16T09:59:15,553 DEBUG [RS:0;037c58a19ddb:37267-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/f3ed23ac403772e0fe4ca9ee892544ec/.tmp/A/12aace3b248347fc94ca8f8143923c21 as hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/f3ed23ac403772e0fe4ca9ee892544ec/A/12aace3b248347fc94ca8f8143923c21 2024-11-16T09:59:15,558 INFO [RS:0;037c58a19ddb:37267-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in f3ed23ac403772e0fe4ca9ee892544ec/B of f3ed23ac403772e0fe4ca9ee892544ec into 5d60f3c9b044415cbf7981c5183581ba(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-16T09:59:15,558 INFO [RS:0;037c58a19ddb:37267-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in f3ed23ac403772e0fe4ca9ee892544ec/A of f3ed23ac403772e0fe4ca9ee892544ec into 12aace3b248347fc94ca8f8143923c21(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-16T09:59:15,558 DEBUG [RS:0;037c58a19ddb:37267-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for f3ed23ac403772e0fe4ca9ee892544ec: 2024-11-16T09:59:15,558 DEBUG [RS:0;037c58a19ddb:37267-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for f3ed23ac403772e0fe4ca9ee892544ec: 2024-11-16T09:59:15,558 INFO [RS:0;037c58a19ddb:37267-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731751137553.f3ed23ac403772e0fe4ca9ee892544ec., storeName=f3ed23ac403772e0fe4ca9ee892544ec/B, priority=13, startTime=1731751155128; duration=0sec 2024-11-16T09:59:15,558 INFO [RS:0;037c58a19ddb:37267-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731751137553.f3ed23ac403772e0fe4ca9ee892544ec., storeName=f3ed23ac403772e0fe4ca9ee892544ec/A, priority=13, startTime=1731751155128; duration=0sec 2024-11-16T09:59:15,558 DEBUG [RS:0;037c58a19ddb:37267-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-16T09:59:15,558 DEBUG [RS:0;037c58a19ddb:37267-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: f3ed23ac403772e0fe4ca9ee892544ec:B 2024-11-16T09:59:15,558 DEBUG [RS:0;037c58a19ddb:37267-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-16T09:59:15,558 DEBUG [RS:0;037c58a19ddb:37267-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-16T09:59:15,559 DEBUG [RS:0;037c58a19ddb:37267-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: f3ed23ac403772e0fe4ca9ee892544ec:A 2024-11-16T09:59:15,559 DEBUG [RS:0;037c58a19ddb:37267-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37333 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-16T09:59:15,559 DEBUG [RS:0;037c58a19ddb:37267-longCompactions-0 {}] regionserver.HStore(1541): f3ed23ac403772e0fe4ca9ee892544ec/C is initiating minor compaction (all files) 2024-11-16T09:59:15,559 INFO [RS:0;037c58a19ddb:37267-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of f3ed23ac403772e0fe4ca9ee892544ec/C in TestAcidGuarantees,,1731751137553.f3ed23ac403772e0fe4ca9ee892544ec. 2024-11-16T09:59:15,559 INFO [RS:0;037c58a19ddb:37267-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/f3ed23ac403772e0fe4ca9ee892544ec/C/1278f4f12e514b53ba17c59c2c5fa75c, hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/f3ed23ac403772e0fe4ca9ee892544ec/C/04b64f8be1d64da9880fa2578af827ac, hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/f3ed23ac403772e0fe4ca9ee892544ec/C/246486ae295b4ce6b3d0df8dfcebfe51] into tmpdir=hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/f3ed23ac403772e0fe4ca9ee892544ec/.tmp, totalSize=36.5 K 2024-11-16T09:59:15,560 DEBUG [RS:0;037c58a19ddb:37267-longCompactions-0 {}] compactions.Compactor(225): Compacting 1278f4f12e514b53ba17c59c2c5fa75c, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=248, earliestPutTs=1731751152043 2024-11-16T09:59:15,560 DEBUG [RS:0;037c58a19ddb:37267-longCompactions-0 {}] compactions.Compactor(225): Compacting 04b64f8be1d64da9880fa2578af827ac, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=274, earliestPutTs=1731751152070 2024-11-16T09:59:15,560 DEBUG [RS:0;037c58a19ddb:37267-longCompactions-0 {}] compactions.Compactor(225): Compacting 246486ae295b4ce6b3d0df8dfcebfe51, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=290, earliestPutTs=1731751153227 2024-11-16T09:59:15,566 INFO [RS:0;037c58a19ddb:37267-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): f3ed23ac403772e0fe4ca9ee892544ec#C#compaction#274 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-16T09:59:15,567 DEBUG [RS:0;037c58a19ddb:37267-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/f3ed23ac403772e0fe4ca9ee892544ec/.tmp/C/42f37ce77c114494ab1505a0d320671a is 50, key is test_row_0/C:col10/1731751153865/Put/seqid=0 2024-11-16T09:59:15,571 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37591 is added to blk_1073742143_1319 (size=12983) 2024-11-16T09:59:15,687 INFO [RS_FLUSH_OPERATIONS-regionserver/037c58a19ddb:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=313 (bloomFilter=true), to=hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/f3ed23ac403772e0fe4ca9ee892544ec/.tmp/B/9c2850d44afa4fc7baca1f5aa115bb38 2024-11-16T09:59:15,695 DEBUG [RS_FLUSH_OPERATIONS-regionserver/037c58a19ddb:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/f3ed23ac403772e0fe4ca9ee892544ec/.tmp/C/9b15a7dac9a24c048f9b79c39fbc2aad is 50, key is test_row_0/C:col10/1731751153887/Put/seqid=0 2024-11-16T09:59:15,705 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37591 is added to blk_1073742144_1320 (size=12301) 2024-11-16T09:59:15,942 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37877 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=74 2024-11-16T09:59:15,979 DEBUG [RS:0;037c58a19ddb:37267-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/f3ed23ac403772e0fe4ca9ee892544ec/.tmp/C/42f37ce77c114494ab1505a0d320671a as hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/f3ed23ac403772e0fe4ca9ee892544ec/C/42f37ce77c114494ab1505a0d320671a 2024-11-16T09:59:15,984 INFO [RS:0;037c58a19ddb:37267-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in f3ed23ac403772e0fe4ca9ee892544ec/C of f3ed23ac403772e0fe4ca9ee892544ec into 42f37ce77c114494ab1505a0d320671a(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-16T09:59:15,984 DEBUG [RS:0;037c58a19ddb:37267-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for f3ed23ac403772e0fe4ca9ee892544ec: 2024-11-16T09:59:15,984 INFO [RS:0;037c58a19ddb:37267-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731751137553.f3ed23ac403772e0fe4ca9ee892544ec., storeName=f3ed23ac403772e0fe4ca9ee892544ec/C, priority=13, startTime=1731751155128; duration=0sec 2024-11-16T09:59:15,984 DEBUG [RS:0;037c58a19ddb:37267-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-16T09:59:15,984 DEBUG [RS:0;037c58a19ddb:37267-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: f3ed23ac403772e0fe4ca9ee892544ec:C 2024-11-16T09:59:16,034 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] regionserver.HRegion(8855): Flush requested on f3ed23ac403772e0fe4ca9ee892544ec 2024-11-16T09:59:16,034 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731751137553.f3ed23ac403772e0fe4ca9ee892544ec. as already flushing 2024-11-16T09:59:16,047 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3ed23ac403772e0fe4ca9ee892544ec, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T09:59:16,047 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] ipc.CallRunner(138): callId: 100 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34414 deadline: 1731751216043, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3ed23ac403772e0fe4ca9ee892544ec, server=037c58a19ddb,37267,1731751077844 2024-11-16T09:59:16,047 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3ed23ac403772e0fe4ca9ee892544ec, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T09:59:16,048 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3ed23ac403772e0fe4ca9ee892544ec, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T09:59:16,048 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] ipc.CallRunner(138): callId: 104 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34408 deadline: 1731751216043, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3ed23ac403772e0fe4ca9ee892544ec, server=037c58a19ddb,37267,1731751077844 2024-11-16T09:59:16,048 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] ipc.CallRunner(138): callId: 108 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34368 deadline: 1731751216043, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3ed23ac403772e0fe4ca9ee892544ec, server=037c58a19ddb,37267,1731751077844 2024-11-16T09:59:16,048 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731751137553.f3ed23ac403772e0fe4ca9ee892544ec., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 , the old value is region=TestAcidGuarantees,,1731751137553.f3ed23ac403772e0fe4ca9ee892544ec., hostname=037c58a19ddb,37267,1731751077844, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3ed23ac403772e0fe4ca9ee892544ec, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:59:16,048 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731751137553.f3ed23ac403772e0fe4ca9ee892544ec., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3ed23ac403772e0fe4ca9ee892544ec, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:59:16,048 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731751137553.f3ed23ac403772e0fe4ca9ee892544ec., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 because the exception is null or not the one we care about 2024-11-16T09:59:16,048 WARN [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRpcRetryingCaller(177): Call to 037c58a19ddb,37267,1731751077844 for 'test_row_0' in f3ed23ac403772e0fe4ca9ee892544ec of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2159 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3ed23ac403772e0fe4ca9ee892544ec, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor44.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3ed23ac403772e0fe4ca9ee892544ec, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-16T09:59:16,048 WARN [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRpcRetryingCaller(177): Call to 037c58a19ddb,37267,1731751077844 for 'test_row_1' in f3ed23ac403772e0fe4ca9ee892544ec of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2158 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3ed23ac403772e0fe4ca9ee892544ec, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor44.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3ed23ac403772e0fe4ca9ee892544ec, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-16T09:59:16,049 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731751137553.f3ed23ac403772e0fe4ca9ee892544ec., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 , the old value is region=TestAcidGuarantees,,1731751137553.f3ed23ac403772e0fe4ca9ee892544ec., hostname=037c58a19ddb,37267,1731751077844, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3ed23ac403772e0fe4ca9ee892544ec, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:59:16,049 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731751137553.f3ed23ac403772e0fe4ca9ee892544ec., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 , the old value is region=TestAcidGuarantees,,1731751137553.f3ed23ac403772e0fe4ca9ee892544ec., hostname=037c58a19ddb,37267,1731751077844, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3ed23ac403772e0fe4ca9ee892544ec, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:59:16,049 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731751137553.f3ed23ac403772e0fe4ca9ee892544ec., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3ed23ac403772e0fe4ca9ee892544ec, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:59:16,049 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731751137553.f3ed23ac403772e0fe4ca9ee892544ec., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3ed23ac403772e0fe4ca9ee892544ec, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:59:16,049 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731751137553.f3ed23ac403772e0fe4ca9ee892544ec., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 because the exception is null or not the one we care about 2024-11-16T09:59:16,049 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731751137553.f3ed23ac403772e0fe4ca9ee892544ec., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 because the exception is null or not the one we care about 2024-11-16T09:59:16,051 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3ed23ac403772e0fe4ca9ee892544ec, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T09:59:16,051 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] ipc.CallRunner(138): callId: 106 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34386 deadline: 1731751216047, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3ed23ac403772e0fe4ca9ee892544ec, server=037c58a19ddb,37267,1731751077844 2024-11-16T09:59:16,051 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3ed23ac403772e0fe4ca9ee892544ec, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T09:59:16,052 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] ipc.CallRunner(138): callId: 101 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34382 deadline: 1731751216048, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3ed23ac403772e0fe4ca9ee892544ec, server=037c58a19ddb,37267,1731751077844 2024-11-16T09:59:16,052 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731751137553.f3ed23ac403772e0fe4ca9ee892544ec., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 , the old value is region=TestAcidGuarantees,,1731751137553.f3ed23ac403772e0fe4ca9ee892544ec., hostname=037c58a19ddb,37267,1731751077844, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3ed23ac403772e0fe4ca9ee892544ec, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:59:16,052 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731751137553.f3ed23ac403772e0fe4ca9ee892544ec., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3ed23ac403772e0fe4ca9ee892544ec, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:59:16,052 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731751137553.f3ed23ac403772e0fe4ca9ee892544ec., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 because the exception is null or not the one we care about 2024-11-16T09:59:16,052 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731751137553.f3ed23ac403772e0fe4ca9ee892544ec., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 , the old value is region=TestAcidGuarantees,,1731751137553.f3ed23ac403772e0fe4ca9ee892544ec., hostname=037c58a19ddb,37267,1731751077844, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3ed23ac403772e0fe4ca9ee892544ec, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:59:16,052 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731751137553.f3ed23ac403772e0fe4ca9ee892544ec., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3ed23ac403772e0fe4ca9ee892544ec, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:59:16,052 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731751137553.f3ed23ac403772e0fe4ca9ee892544ec., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 because the exception is null or not the one we care about 2024-11-16T09:59:16,106 INFO [RS_FLUSH_OPERATIONS-regionserver/037c58a19ddb:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=313 (bloomFilter=true), to=hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/f3ed23ac403772e0fe4ca9ee892544ec/.tmp/C/9b15a7dac9a24c048f9b79c39fbc2aad 2024-11-16T09:59:16,111 DEBUG [RS_FLUSH_OPERATIONS-regionserver/037c58a19ddb:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/f3ed23ac403772e0fe4ca9ee892544ec/.tmp/A/b8255e158316461c9e6faf3a1fcf57f1 as hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/f3ed23ac403772e0fe4ca9ee892544ec/A/b8255e158316461c9e6faf3a1fcf57f1 2024-11-16T09:59:16,115 INFO [RS_FLUSH_OPERATIONS-regionserver/037c58a19ddb:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/f3ed23ac403772e0fe4ca9ee892544ec/A/b8255e158316461c9e6faf3a1fcf57f1, entries=150, sequenceid=313, filesize=12.0 K 2024-11-16T09:59:16,116 DEBUG [RS_FLUSH_OPERATIONS-regionserver/037c58a19ddb:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/f3ed23ac403772e0fe4ca9ee892544ec/.tmp/B/9c2850d44afa4fc7baca1f5aa115bb38 as hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/f3ed23ac403772e0fe4ca9ee892544ec/B/9c2850d44afa4fc7baca1f5aa115bb38 2024-11-16T09:59:16,120 INFO [RS_FLUSH_OPERATIONS-regionserver/037c58a19ddb:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/f3ed23ac403772e0fe4ca9ee892544ec/B/9c2850d44afa4fc7baca1f5aa115bb38, entries=150, sequenceid=313, filesize=12.0 K 2024-11-16T09:59:16,121 DEBUG [RS_FLUSH_OPERATIONS-regionserver/037c58a19ddb:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/f3ed23ac403772e0fe4ca9ee892544ec/.tmp/C/9b15a7dac9a24c048f9b79c39fbc2aad as hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/f3ed23ac403772e0fe4ca9ee892544ec/C/9b15a7dac9a24c048f9b79c39fbc2aad 2024-11-16T09:59:16,124 INFO [RS_FLUSH_OPERATIONS-regionserver/037c58a19ddb:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/f3ed23ac403772e0fe4ca9ee892544ec/C/9b15a7dac9a24c048f9b79c39fbc2aad, entries=150, sequenceid=313, filesize=12.0 K 2024-11-16T09:59:16,125 INFO [RS_FLUSH_OPERATIONS-regionserver/037c58a19ddb:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HRegion(3140): Finished flush of dataSize ~134.18 KB/137400, heapSize ~352.27 KB/360720, currentSize=67.09 KB/68700 for f3ed23ac403772e0fe4ca9ee892544ec in 860ms, sequenceid=313, compaction requested=false 2024-11-16T09:59:16,125 DEBUG [RS_FLUSH_OPERATIONS-regionserver/037c58a19ddb:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HRegion(2603): Flush status journal for f3ed23ac403772e0fe4ca9ee892544ec: 2024-11-16T09:59:16,125 DEBUG [RS_FLUSH_OPERATIONS-regionserver/037c58a19ddb:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731751137553.f3ed23ac403772e0fe4ca9ee892544ec. 2024-11-16T09:59:16,125 DEBUG [RS_FLUSH_OPERATIONS-regionserver/037c58a19ddb:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=75 2024-11-16T09:59:16,126 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37877 {}] master.HMaster(4169): Remote procedure done, pid=75 2024-11-16T09:59:16,129 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=75, resume processing ppid=74 2024-11-16T09:59:16,129 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=75, ppid=74, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.3210 sec 2024-11-16T09:59:16,131 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=74, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=74, table=TestAcidGuarantees in 1.3270 sec 2024-11-16T09:59:16,155 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] regionserver.HRegion(8855): Flush requested on f3ed23ac403772e0fe4ca9ee892544ec 2024-11-16T09:59:16,156 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-16T09:59:16,156 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing f3ed23ac403772e0fe4ca9ee892544ec 3/3 column families, dataSize=73.80 KB heapSize=194.11 KB 2024-11-16T09:59:16,160 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/f3ed23ac403772e0fe4ca9ee892544ec/.tmp/A/7edbb0569e5c4e04bfa1b45acf180081 is 50, key is test_row_0/A:col10/1731751156154/Put/seqid=0 2024-11-16T09:59:16,165 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37591 is added to blk_1073742145_1321 (size=14741) 2024-11-16T09:59:16,193 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3ed23ac403772e0fe4ca9ee892544ec, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T09:59:16,193 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] ipc.CallRunner(138): callId: 107 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34382 deadline: 1731751216189, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3ed23ac403772e0fe4ca9ee892544ec, server=037c58a19ddb,37267,1731751077844 2024-11-16T09:59:16,194 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3ed23ac403772e0fe4ca9ee892544ec, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T09:59:16,194 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] ipc.CallRunner(138): callId: 118 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34368 deadline: 1731751216190, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3ed23ac403772e0fe4ca9ee892544ec, server=037c58a19ddb,37267,1731751077844 2024-11-16T09:59:16,194 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731751137553.f3ed23ac403772e0fe4ca9ee892544ec., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 , the old value is region=TestAcidGuarantees,,1731751137553.f3ed23ac403772e0fe4ca9ee892544ec., hostname=037c58a19ddb,37267,1731751077844, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3ed23ac403772e0fe4ca9ee892544ec, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:59:16,194 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731751137553.f3ed23ac403772e0fe4ca9ee892544ec., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3ed23ac403772e0fe4ca9ee892544ec, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:59:16,194 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731751137553.f3ed23ac403772e0fe4ca9ee892544ec., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 because the exception is null or not the one we care about 2024-11-16T09:59:16,194 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731751137553.f3ed23ac403772e0fe4ca9ee892544ec., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 , the old value is region=TestAcidGuarantees,,1731751137553.f3ed23ac403772e0fe4ca9ee892544ec., hostname=037c58a19ddb,37267,1731751077844, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3ed23ac403772e0fe4ca9ee892544ec, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:59:16,194 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731751137553.f3ed23ac403772e0fe4ca9ee892544ec., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3ed23ac403772e0fe4ca9ee892544ec, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:59:16,194 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731751137553.f3ed23ac403772e0fe4ca9ee892544ec., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 because the exception is null or not the one we care about 2024-11-16T09:59:16,194 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3ed23ac403772e0fe4ca9ee892544ec, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T09:59:16,195 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] ipc.CallRunner(138): callId: 113 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34386 deadline: 1731751216191, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3ed23ac403772e0fe4ca9ee892544ec, server=037c58a19ddb,37267,1731751077844 2024-11-16T09:59:16,195 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731751137553.f3ed23ac403772e0fe4ca9ee892544ec., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 , the old value is region=TestAcidGuarantees,,1731751137553.f3ed23ac403772e0fe4ca9ee892544ec., hostname=037c58a19ddb,37267,1731751077844, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3ed23ac403772e0fe4ca9ee892544ec, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:59:16,195 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731751137553.f3ed23ac403772e0fe4ca9ee892544ec., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3ed23ac403772e0fe4ca9ee892544ec, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:59:16,195 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731751137553.f3ed23ac403772e0fe4ca9ee892544ec., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 because the exception is null or not the one we care about 2024-11-16T09:59:16,305 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3ed23ac403772e0fe4ca9ee892544ec, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T09:59:16,305 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] ipc.CallRunner(138): callId: 108 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34382 deadline: 1731751216302, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3ed23ac403772e0fe4ca9ee892544ec, server=037c58a19ddb,37267,1731751077844 2024-11-16T09:59:16,306 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3ed23ac403772e0fe4ca9ee892544ec, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T09:59:16,306 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731751137553.f3ed23ac403772e0fe4ca9ee892544ec., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 , the old value is region=TestAcidGuarantees,,1731751137553.f3ed23ac403772e0fe4ca9ee892544ec., hostname=037c58a19ddb,37267,1731751077844, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3ed23ac403772e0fe4ca9ee892544ec, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:59:16,306 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731751137553.f3ed23ac403772e0fe4ca9ee892544ec., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3ed23ac403772e0fe4ca9ee892544ec, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:59:16,306 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] ipc.CallRunner(138): callId: 119 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34368 deadline: 1731751216302, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3ed23ac403772e0fe4ca9ee892544ec, server=037c58a19ddb,37267,1731751077844 2024-11-16T09:59:16,306 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731751137553.f3ed23ac403772e0fe4ca9ee892544ec., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 because the exception is null or not the one we care about 2024-11-16T09:59:16,306 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3ed23ac403772e0fe4ca9ee892544ec, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T09:59:16,306 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731751137553.f3ed23ac403772e0fe4ca9ee892544ec., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 , the old value is region=TestAcidGuarantees,,1731751137553.f3ed23ac403772e0fe4ca9ee892544ec., hostname=037c58a19ddb,37267,1731751077844, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3ed23ac403772e0fe4ca9ee892544ec, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:59:16,306 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731751137553.f3ed23ac403772e0fe4ca9ee892544ec., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3ed23ac403772e0fe4ca9ee892544ec, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:59:16,306 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] ipc.CallRunner(138): callId: 114 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34386 deadline: 1731751216302, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3ed23ac403772e0fe4ca9ee892544ec, server=037c58a19ddb,37267,1731751077844 2024-11-16T09:59:16,306 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731751137553.f3ed23ac403772e0fe4ca9ee892544ec., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 because the exception is null or not the one we care about 2024-11-16T09:59:16,306 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731751137553.f3ed23ac403772e0fe4ca9ee892544ec., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 , the old value is region=TestAcidGuarantees,,1731751137553.f3ed23ac403772e0fe4ca9ee892544ec., hostname=037c58a19ddb,37267,1731751077844, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3ed23ac403772e0fe4ca9ee892544ec, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:59:16,306 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731751137553.f3ed23ac403772e0fe4ca9ee892544ec., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3ed23ac403772e0fe4ca9ee892544ec, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:59:16,306 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731751137553.f3ed23ac403772e0fe4ca9ee892544ec., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 because the exception is null or not the one we care about 2024-11-16T09:59:16,516 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3ed23ac403772e0fe4ca9ee892544ec, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T09:59:16,516 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3ed23ac403772e0fe4ca9ee892544ec, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T09:59:16,517 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] ipc.CallRunner(138): callId: 109 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34382 deadline: 1731751216513, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3ed23ac403772e0fe4ca9ee892544ec, server=037c58a19ddb,37267,1731751077844 2024-11-16T09:59:16,517 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] ipc.CallRunner(138): callId: 120 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34368 deadline: 1731751216513, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3ed23ac403772e0fe4ca9ee892544ec, server=037c58a19ddb,37267,1731751077844 2024-11-16T09:59:16,517 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3ed23ac403772e0fe4ca9ee892544ec, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T09:59:16,517 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731751137553.f3ed23ac403772e0fe4ca9ee892544ec., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 , the old value is region=TestAcidGuarantees,,1731751137553.f3ed23ac403772e0fe4ca9ee892544ec., hostname=037c58a19ddb,37267,1731751077844, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3ed23ac403772e0fe4ca9ee892544ec, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:59:16,517 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] ipc.CallRunner(138): callId: 115 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34386 deadline: 1731751216514, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3ed23ac403772e0fe4ca9ee892544ec, server=037c58a19ddb,37267,1731751077844 2024-11-16T09:59:16,517 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731751137553.f3ed23ac403772e0fe4ca9ee892544ec., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3ed23ac403772e0fe4ca9ee892544ec, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:59:16,517 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731751137553.f3ed23ac403772e0fe4ca9ee892544ec., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 because the exception is null or not the one we care about 2024-11-16T09:59:16,517 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731751137553.f3ed23ac403772e0fe4ca9ee892544ec., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 , the old value is region=TestAcidGuarantees,,1731751137553.f3ed23ac403772e0fe4ca9ee892544ec., hostname=037c58a19ddb,37267,1731751077844, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3ed23ac403772e0fe4ca9ee892544ec, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:59:16,517 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731751137553.f3ed23ac403772e0fe4ca9ee892544ec., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3ed23ac403772e0fe4ca9ee892544ec, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:59:16,517 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731751137553.f3ed23ac403772e0fe4ca9ee892544ec., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 because the exception is null or not the one we care about 2024-11-16T09:59:16,517 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731751137553.f3ed23ac403772e0fe4ca9ee892544ec., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 , the old value is region=TestAcidGuarantees,,1731751137553.f3ed23ac403772e0fe4ca9ee892544ec., hostname=037c58a19ddb,37267,1731751077844, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3ed23ac403772e0fe4ca9ee892544ec, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:59:16,517 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731751137553.f3ed23ac403772e0fe4ca9ee892544ec., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3ed23ac403772e0fe4ca9ee892544ec, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:59:16,517 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731751137553.f3ed23ac403772e0fe4ca9ee892544ec., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 because the exception is null or not the one we care about 2024-11-16T09:59:16,566 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=330 (bloomFilter=true), to=hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/f3ed23ac403772e0fe4ca9ee892544ec/.tmp/A/7edbb0569e5c4e04bfa1b45acf180081 2024-11-16T09:59:16,574 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/f3ed23ac403772e0fe4ca9ee892544ec/.tmp/B/e5ef6c31fd4d413bab0b528ff6c5762b is 50, key is test_row_0/B:col10/1731751156154/Put/seqid=0 2024-11-16T09:59:16,578 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37591 is added to blk_1073742146_1322 (size=12301) 2024-11-16T09:59:16,826 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3ed23ac403772e0fe4ca9ee892544ec, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T09:59:16,826 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] ipc.CallRunner(138): callId: 121 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34368 deadline: 1731751216823, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3ed23ac403772e0fe4ca9ee892544ec, server=037c58a19ddb,37267,1731751077844 2024-11-16T09:59:16,826 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3ed23ac403772e0fe4ca9ee892544ec, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T09:59:16,826 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731751137553.f3ed23ac403772e0fe4ca9ee892544ec., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 , the old value is region=TestAcidGuarantees,,1731751137553.f3ed23ac403772e0fe4ca9ee892544ec., hostname=037c58a19ddb,37267,1731751077844, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3ed23ac403772e0fe4ca9ee892544ec, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:59:16,826 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731751137553.f3ed23ac403772e0fe4ca9ee892544ec., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3ed23ac403772e0fe4ca9ee892544ec, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:59:16,826 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731751137553.f3ed23ac403772e0fe4ca9ee892544ec., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 because the exception is null or not the one we care about 2024-11-16T09:59:16,826 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] ipc.CallRunner(138): callId: 116 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34386 deadline: 1731751216823, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3ed23ac403772e0fe4ca9ee892544ec, server=037c58a19ddb,37267,1731751077844 2024-11-16T09:59:16,826 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3ed23ac403772e0fe4ca9ee892544ec, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T09:59:16,827 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] ipc.CallRunner(138): callId: 110 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34382 deadline: 1731751216823, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3ed23ac403772e0fe4ca9ee892544ec, server=037c58a19ddb,37267,1731751077844 2024-11-16T09:59:16,827 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731751137553.f3ed23ac403772e0fe4ca9ee892544ec., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 , the old value is region=TestAcidGuarantees,,1731751137553.f3ed23ac403772e0fe4ca9ee892544ec., hostname=037c58a19ddb,37267,1731751077844, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3ed23ac403772e0fe4ca9ee892544ec, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:59:16,827 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731751137553.f3ed23ac403772e0fe4ca9ee892544ec., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3ed23ac403772e0fe4ca9ee892544ec, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:59:16,827 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731751137553.f3ed23ac403772e0fe4ca9ee892544ec., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 , the old value is region=TestAcidGuarantees,,1731751137553.f3ed23ac403772e0fe4ca9ee892544ec., hostname=037c58a19ddb,37267,1731751077844, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3ed23ac403772e0fe4ca9ee892544ec, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:59:16,827 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731751137553.f3ed23ac403772e0fe4ca9ee892544ec., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 because the exception is null or not the one we care about 2024-11-16T09:59:16,827 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731751137553.f3ed23ac403772e0fe4ca9ee892544ec., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3ed23ac403772e0fe4ca9ee892544ec, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:59:16,827 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731751137553.f3ed23ac403772e0fe4ca9ee892544ec., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 because the exception is null or not the one we care about 2024-11-16T09:59:16,953 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37877 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=74 2024-11-16T09:59:16,953 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-16T09:59:16,953 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37877 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-16T09:59:16,955 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37877 {}] procedure2.ProcedureExecutor(1139): Stored pid=76, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=76, table=TestAcidGuarantees 2024-11-16T09:59:16,955 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37877 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=76 2024-11-16T09:59:16,955 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=76, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=76, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-16T09:59:16,956 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=76, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=76, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-16T09:59:16,956 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=77, ppid=76, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-16T09:59:16,979 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=330 (bloomFilter=true), to=hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/f3ed23ac403772e0fe4ca9ee892544ec/.tmp/B/e5ef6c31fd4d413bab0b528ff6c5762b 2024-11-16T09:59:16,987 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/f3ed23ac403772e0fe4ca9ee892544ec/.tmp/C/24ccf25c8a5a4d6e8f0d6ebdf95b6954 is 50, key is test_row_0/C:col10/1731751156154/Put/seqid=0 2024-11-16T09:59:16,991 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37591 is added to blk_1073742147_1323 (size=12301) 2024-11-16T09:59:17,062 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37877 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=76 2024-11-16T09:59:17,109 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=37267 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=77 2024-11-16T09:59:17,109 DEBUG [RS_FLUSH_OPERATIONS-regionserver/037c58a19ddb:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731751137553.f3ed23ac403772e0fe4ca9ee892544ec. 2024-11-16T09:59:17,109 DEBUG [RS_FLUSH_OPERATIONS-regionserver/037c58a19ddb:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731751137553.f3ed23ac403772e0fe4ca9ee892544ec. as already flushing 2024-11-16T09:59:17,109 DEBUG [RS_FLUSH_OPERATIONS-regionserver/037c58a19ddb:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731751137553.f3ed23ac403772e0fe4ca9ee892544ec. 2024-11-16T09:59:17,109 ERROR [RS_FLUSH_OPERATIONS-regionserver/037c58a19ddb:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] handler.RSProcedureHandler(58): pid=77 java.io.IOException: Unable to complete flush {ENCODED => f3ed23ac403772e0fe4ca9ee892544ec, NAME => 'TestAcidGuarantees,,1731751137553.f3ed23ac403772e0fe4ca9ee892544ec.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T09:59:17,110 DEBUG [RS_FLUSH_OPERATIONS-regionserver/037c58a19ddb:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=77 java.io.IOException: Unable to complete flush {ENCODED => f3ed23ac403772e0fe4ca9ee892544ec, NAME => 'TestAcidGuarantees,,1731751137553.f3ed23ac403772e0fe4ca9ee892544ec.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T09:59:17,110 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37877 {}] master.HMaster(4177): Remote procedure failed, pid=77 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => f3ed23ac403772e0fe4ca9ee892544ec, NAME => 'TestAcidGuarantees,,1731751137553.f3ed23ac403772e0fe4ca9ee892544ec.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => f3ed23ac403772e0fe4ca9ee892544ec, NAME => 'TestAcidGuarantees,,1731751137553.f3ed23ac403772e0fe4ca9ee892544ec.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T09:59:17,262 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=37267 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=77 2024-11-16T09:59:17,262 DEBUG [RS_FLUSH_OPERATIONS-regionserver/037c58a19ddb:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731751137553.f3ed23ac403772e0fe4ca9ee892544ec. 2024-11-16T09:59:17,263 DEBUG [RS_FLUSH_OPERATIONS-regionserver/037c58a19ddb:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731751137553.f3ed23ac403772e0fe4ca9ee892544ec. as already flushing 2024-11-16T09:59:17,263 DEBUG [RS_FLUSH_OPERATIONS-regionserver/037c58a19ddb:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731751137553.f3ed23ac403772e0fe4ca9ee892544ec. 2024-11-16T09:59:17,263 ERROR [RS_FLUSH_OPERATIONS-regionserver/037c58a19ddb:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] handler.RSProcedureHandler(58): pid=77 java.io.IOException: Unable to complete flush {ENCODED => f3ed23ac403772e0fe4ca9ee892544ec, NAME => 'TestAcidGuarantees,,1731751137553.f3ed23ac403772e0fe4ca9ee892544ec.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T09:59:17,263 DEBUG [RS_FLUSH_OPERATIONS-regionserver/037c58a19ddb:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=77 java.io.IOException: Unable to complete flush {ENCODED => f3ed23ac403772e0fe4ca9ee892544ec, NAME => 'TestAcidGuarantees,,1731751137553.f3ed23ac403772e0fe4ca9ee892544ec.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T09:59:17,264 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37877 {}] master.HMaster(4177): Remote procedure failed, pid=77 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => f3ed23ac403772e0fe4ca9ee892544ec, NAME => 'TestAcidGuarantees,,1731751137553.f3ed23ac403772e0fe4ca9ee892544ec.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => f3ed23ac403772e0fe4ca9ee892544ec, NAME => 'TestAcidGuarantees,,1731751137553.f3ed23ac403772e0fe4ca9ee892544ec.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T09:59:17,273 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37877 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=76 2024-11-16T09:59:17,335 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3ed23ac403772e0fe4ca9ee892544ec, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T09:59:17,336 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] ipc.CallRunner(138): callId: 122 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34368 deadline: 1731751217333, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3ed23ac403772e0fe4ca9ee892544ec, server=037c58a19ddb,37267,1731751077844 2024-11-16T09:59:17,336 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3ed23ac403772e0fe4ca9ee892544ec, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T09:59:17,336 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731751137553.f3ed23ac403772e0fe4ca9ee892544ec., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 , the old value is region=TestAcidGuarantees,,1731751137553.f3ed23ac403772e0fe4ca9ee892544ec., hostname=037c58a19ddb,37267,1731751077844, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3ed23ac403772e0fe4ca9ee892544ec, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:59:17,336 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731751137553.f3ed23ac403772e0fe4ca9ee892544ec., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3ed23ac403772e0fe4ca9ee892544ec, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:59:17,336 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] ipc.CallRunner(138): callId: 117 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34386 deadline: 1731751217333, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3ed23ac403772e0fe4ca9ee892544ec, server=037c58a19ddb,37267,1731751077844 2024-11-16T09:59:17,336 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731751137553.f3ed23ac403772e0fe4ca9ee892544ec., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 because the exception is null or not the one we care about 2024-11-16T09:59:17,336 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731751137553.f3ed23ac403772e0fe4ca9ee892544ec., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 , the old value is region=TestAcidGuarantees,,1731751137553.f3ed23ac403772e0fe4ca9ee892544ec., hostname=037c58a19ddb,37267,1731751077844, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3ed23ac403772e0fe4ca9ee892544ec, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:59:17,336 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3ed23ac403772e0fe4ca9ee892544ec, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T09:59:17,336 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731751137553.f3ed23ac403772e0fe4ca9ee892544ec., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3ed23ac403772e0fe4ca9ee892544ec, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:59:17,336 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731751137553.f3ed23ac403772e0fe4ca9ee892544ec., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 because the exception is null or not the one we care about 2024-11-16T09:59:17,336 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] ipc.CallRunner(138): callId: 111 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34382 deadline: 1731751217333, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3ed23ac403772e0fe4ca9ee892544ec, server=037c58a19ddb,37267,1731751077844 2024-11-16T09:59:17,337 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731751137553.f3ed23ac403772e0fe4ca9ee892544ec., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 , the old value is region=TestAcidGuarantees,,1731751137553.f3ed23ac403772e0fe4ca9ee892544ec., hostname=037c58a19ddb,37267,1731751077844, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3ed23ac403772e0fe4ca9ee892544ec, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:59:17,337 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731751137553.f3ed23ac403772e0fe4ca9ee892544ec., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3ed23ac403772e0fe4ca9ee892544ec, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:59:17,337 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731751137553.f3ed23ac403772e0fe4ca9ee892544ec., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 because the exception is null or not the one we care about 2024-11-16T09:59:17,391 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=330 (bloomFilter=true), to=hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/f3ed23ac403772e0fe4ca9ee892544ec/.tmp/C/24ccf25c8a5a4d6e8f0d6ebdf95b6954 2024-11-16T09:59:17,396 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/f3ed23ac403772e0fe4ca9ee892544ec/.tmp/A/7edbb0569e5c4e04bfa1b45acf180081 as hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/f3ed23ac403772e0fe4ca9ee892544ec/A/7edbb0569e5c4e04bfa1b45acf180081 2024-11-16T09:59:17,400 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/f3ed23ac403772e0fe4ca9ee892544ec/A/7edbb0569e5c4e04bfa1b45acf180081, entries=200, sequenceid=330, filesize=14.4 K 2024-11-16T09:59:17,401 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/f3ed23ac403772e0fe4ca9ee892544ec/.tmp/B/e5ef6c31fd4d413bab0b528ff6c5762b as hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/f3ed23ac403772e0fe4ca9ee892544ec/B/e5ef6c31fd4d413bab0b528ff6c5762b 2024-11-16T09:59:17,404 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/f3ed23ac403772e0fe4ca9ee892544ec/B/e5ef6c31fd4d413bab0b528ff6c5762b, entries=150, sequenceid=330, filesize=12.0 K 2024-11-16T09:59:17,406 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/f3ed23ac403772e0fe4ca9ee892544ec/.tmp/C/24ccf25c8a5a4d6e8f0d6ebdf95b6954 as hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/f3ed23ac403772e0fe4ca9ee892544ec/C/24ccf25c8a5a4d6e8f0d6ebdf95b6954 2024-11-16T09:59:17,411 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/f3ed23ac403772e0fe4ca9ee892544ec/C/24ccf25c8a5a4d6e8f0d6ebdf95b6954, entries=150, sequenceid=330, filesize=12.0 K 2024-11-16T09:59:17,413 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~73.80 KB/75570, heapSize ~194.06 KB/198720, currentSize=127.47 KB/130530 for f3ed23ac403772e0fe4ca9ee892544ec in 1257ms, sequenceid=330, compaction requested=true 2024-11-16T09:59:17,413 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for f3ed23ac403772e0fe4ca9ee892544ec: 2024-11-16T09:59:17,413 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store f3ed23ac403772e0fe4ca9ee892544ec:A, priority=-2147483648, current under compaction store size is 1 2024-11-16T09:59:17,413 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-16T09:59:17,413 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store f3ed23ac403772e0fe4ca9ee892544ec:B, priority=-2147483648, current under compaction store size is 2 2024-11-16T09:59:17,413 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-16T09:59:17,413 DEBUG [RS:0;037c58a19ddb:37267-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-16T09:59:17,413 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store f3ed23ac403772e0fe4ca9ee892544ec:C, priority=-2147483648, current under compaction store size is 3 2024-11-16T09:59:17,413 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=2), splitQueue=0 2024-11-16T09:59:17,413 DEBUG [RS:0;037c58a19ddb:37267-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-16T09:59:17,414 DEBUG [RS:0;037c58a19ddb:37267-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37585 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-16T09:59:17,414 DEBUG [RS:0;037c58a19ddb:37267-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 40025 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-16T09:59:17,414 DEBUG [RS:0;037c58a19ddb:37267-shortCompactions-0 {}] regionserver.HStore(1541): f3ed23ac403772e0fe4ca9ee892544ec/B is initiating minor compaction (all files) 2024-11-16T09:59:17,414 DEBUG [RS:0;037c58a19ddb:37267-longCompactions-0 {}] regionserver.HStore(1541): f3ed23ac403772e0fe4ca9ee892544ec/A is initiating minor compaction (all files) 2024-11-16T09:59:17,415 INFO [RS:0;037c58a19ddb:37267-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of f3ed23ac403772e0fe4ca9ee892544ec/B in TestAcidGuarantees,,1731751137553.f3ed23ac403772e0fe4ca9ee892544ec. 2024-11-16T09:59:17,415 INFO [RS:0;037c58a19ddb:37267-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of f3ed23ac403772e0fe4ca9ee892544ec/A in TestAcidGuarantees,,1731751137553.f3ed23ac403772e0fe4ca9ee892544ec. 2024-11-16T09:59:17,415 INFO [RS:0;037c58a19ddb:37267-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/f3ed23ac403772e0fe4ca9ee892544ec/B/5d60f3c9b044415cbf7981c5183581ba, hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/f3ed23ac403772e0fe4ca9ee892544ec/B/9c2850d44afa4fc7baca1f5aa115bb38, hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/f3ed23ac403772e0fe4ca9ee892544ec/B/e5ef6c31fd4d413bab0b528ff6c5762b] into tmpdir=hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/f3ed23ac403772e0fe4ca9ee892544ec/.tmp, totalSize=36.7 K 2024-11-16T09:59:17,415 INFO [RS:0;037c58a19ddb:37267-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/f3ed23ac403772e0fe4ca9ee892544ec/A/12aace3b248347fc94ca8f8143923c21, hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/f3ed23ac403772e0fe4ca9ee892544ec/A/b8255e158316461c9e6faf3a1fcf57f1, hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/f3ed23ac403772e0fe4ca9ee892544ec/A/7edbb0569e5c4e04bfa1b45acf180081] into tmpdir=hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/f3ed23ac403772e0fe4ca9ee892544ec/.tmp, totalSize=39.1 K 2024-11-16T09:59:17,415 DEBUG [RS:0;037c58a19ddb:37267-shortCompactions-0 {}] compactions.Compactor(225): Compacting 5d60f3c9b044415cbf7981c5183581ba, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=290, earliestPutTs=1731751153227 2024-11-16T09:59:17,415 DEBUG [RS:0;037c58a19ddb:37267-longCompactions-0 {}] compactions.Compactor(225): Compacting 12aace3b248347fc94ca8f8143923c21, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=290, earliestPutTs=1731751153227 2024-11-16T09:59:17,416 DEBUG [RS:0;037c58a19ddb:37267-longCompactions-0 {}] compactions.Compactor(225): Compacting b8255e158316461c9e6faf3a1fcf57f1, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=313, earliestPutTs=1731751153879 2024-11-16T09:59:17,416 DEBUG [RS:0;037c58a19ddb:37267-shortCompactions-0 {}] compactions.Compactor(225): Compacting 9c2850d44afa4fc7baca1f5aa115bb38, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=313, earliestPutTs=1731751153879 2024-11-16T09:59:17,416 DEBUG [RS:0;037c58a19ddb:37267-shortCompactions-0 {}] compactions.Compactor(225): Compacting e5ef6c31fd4d413bab0b528ff6c5762b, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=330, earliestPutTs=1731751156045 2024-11-16T09:59:17,416 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=37267 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=77 2024-11-16T09:59:17,417 DEBUG [RS:0;037c58a19ddb:37267-longCompactions-0 {}] compactions.Compactor(225): Compacting 7edbb0569e5c4e04bfa1b45acf180081, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=330, earliestPutTs=1731751156038 2024-11-16T09:59:17,417 DEBUG [RS_FLUSH_OPERATIONS-regionserver/037c58a19ddb:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731751137553.f3ed23ac403772e0fe4ca9ee892544ec. 2024-11-16T09:59:17,417 INFO [RS_FLUSH_OPERATIONS-regionserver/037c58a19ddb:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HRegion(2902): Flushing f3ed23ac403772e0fe4ca9ee892544ec 3/3 column families, dataSize=127.47 KB heapSize=334.73 KB 2024-11-16T09:59:17,430 DEBUG [RS_FLUSH_OPERATIONS-regionserver/037c58a19ddb:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/f3ed23ac403772e0fe4ca9ee892544ec/.tmp/A/d07186c35def4c13966a8948333333ac is 50, key is test_row_0/A:col10/1731751156184/Put/seqid=0 2024-11-16T09:59:17,439 INFO [RS:0;037c58a19ddb:37267-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): f3ed23ac403772e0fe4ca9ee892544ec#B#compaction#280 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-16T09:59:17,439 DEBUG [RS:0;037c58a19ddb:37267-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/f3ed23ac403772e0fe4ca9ee892544ec/.tmp/B/921ee41b9b074ed2b0d2e2c28b2edec3 is 50, key is test_row_0/B:col10/1731751156154/Put/seqid=0 2024-11-16T09:59:17,442 INFO [RS:0;037c58a19ddb:37267-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): f3ed23ac403772e0fe4ca9ee892544ec#A#compaction#281 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-16T09:59:17,442 DEBUG [RS:0;037c58a19ddb:37267-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/f3ed23ac403772e0fe4ca9ee892544ec/.tmp/A/b097f1e4754c45e7863be4f0cb423c62 is 50, key is test_row_0/A:col10/1731751156154/Put/seqid=0 2024-11-16T09:59:17,457 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37591 is added to blk_1073742148_1324 (size=12301) 2024-11-16T09:59:17,458 INFO [RS_FLUSH_OPERATIONS-regionserver/037c58a19ddb:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=352 (bloomFilter=true), to=hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/f3ed23ac403772e0fe4ca9ee892544ec/.tmp/A/d07186c35def4c13966a8948333333ac 2024-11-16T09:59:17,469 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37591 is added to blk_1073742150_1326 (size=13085) 2024-11-16T09:59:17,477 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37591 is added to blk_1073742149_1325 (size=13085) 2024-11-16T09:59:17,478 DEBUG [RS_FLUSH_OPERATIONS-regionserver/037c58a19ddb:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/f3ed23ac403772e0fe4ca9ee892544ec/.tmp/B/7f75e570933e48349567b87f6b6576d1 is 50, key is test_row_0/B:col10/1731751156184/Put/seqid=0 2024-11-16T09:59:17,481 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37591 is added to blk_1073742151_1327 (size=12301) 2024-11-16T09:59:17,482 INFO [RS_FLUSH_OPERATIONS-regionserver/037c58a19ddb:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=352 (bloomFilter=true), to=hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/f3ed23ac403772e0fe4ca9ee892544ec/.tmp/B/7f75e570933e48349567b87f6b6576d1 2024-11-16T09:59:17,488 DEBUG [RS_FLUSH_OPERATIONS-regionserver/037c58a19ddb:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/f3ed23ac403772e0fe4ca9ee892544ec/.tmp/C/76b2417e63f04287845faacef82c7350 is 50, key is test_row_0/C:col10/1731751156184/Put/seqid=0 2024-11-16T09:59:17,497 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37591 is added to blk_1073742152_1328 (size=12301) 2024-11-16T09:59:17,497 INFO [RS_FLUSH_OPERATIONS-regionserver/037c58a19ddb:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=352 (bloomFilter=true), to=hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/f3ed23ac403772e0fe4ca9ee892544ec/.tmp/C/76b2417e63f04287845faacef82c7350 2024-11-16T09:59:17,502 DEBUG [RS_FLUSH_OPERATIONS-regionserver/037c58a19ddb:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/f3ed23ac403772e0fe4ca9ee892544ec/.tmp/A/d07186c35def4c13966a8948333333ac as hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/f3ed23ac403772e0fe4ca9ee892544ec/A/d07186c35def4c13966a8948333333ac 2024-11-16T09:59:17,508 INFO [RS_FLUSH_OPERATIONS-regionserver/037c58a19ddb:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/f3ed23ac403772e0fe4ca9ee892544ec/A/d07186c35def4c13966a8948333333ac, entries=150, sequenceid=352, filesize=12.0 K 2024-11-16T09:59:17,508 DEBUG [RS_FLUSH_OPERATIONS-regionserver/037c58a19ddb:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/f3ed23ac403772e0fe4ca9ee892544ec/.tmp/B/7f75e570933e48349567b87f6b6576d1 as hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/f3ed23ac403772e0fe4ca9ee892544ec/B/7f75e570933e48349567b87f6b6576d1 2024-11-16T09:59:17,512 INFO [RS_FLUSH_OPERATIONS-regionserver/037c58a19ddb:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/f3ed23ac403772e0fe4ca9ee892544ec/B/7f75e570933e48349567b87f6b6576d1, entries=150, sequenceid=352, filesize=12.0 K 2024-11-16T09:59:17,513 DEBUG [RS_FLUSH_OPERATIONS-regionserver/037c58a19ddb:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/f3ed23ac403772e0fe4ca9ee892544ec/.tmp/C/76b2417e63f04287845faacef82c7350 as hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/f3ed23ac403772e0fe4ca9ee892544ec/C/76b2417e63f04287845faacef82c7350 2024-11-16T09:59:17,517 INFO [RS_FLUSH_OPERATIONS-regionserver/037c58a19ddb:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/f3ed23ac403772e0fe4ca9ee892544ec/C/76b2417e63f04287845faacef82c7350, entries=150, sequenceid=352, filesize=12.0 K 2024-11-16T09:59:17,518 INFO [RS_FLUSH_OPERATIONS-regionserver/037c58a19ddb:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HRegion(3140): Finished flush of dataSize ~127.47 KB/130530, heapSize ~334.69 KB/342720, currentSize=0 B/0 for f3ed23ac403772e0fe4ca9ee892544ec in 101ms, sequenceid=352, compaction requested=true 2024-11-16T09:59:17,518 DEBUG [RS_FLUSH_OPERATIONS-regionserver/037c58a19ddb:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HRegion(2603): Flush status journal for f3ed23ac403772e0fe4ca9ee892544ec: 2024-11-16T09:59:17,518 DEBUG [RS_FLUSH_OPERATIONS-regionserver/037c58a19ddb:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731751137553.f3ed23ac403772e0fe4ca9ee892544ec. 2024-11-16T09:59:17,518 DEBUG [RS_FLUSH_OPERATIONS-regionserver/037c58a19ddb:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=77 2024-11-16T09:59:17,518 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37877 {}] master.HMaster(4169): Remote procedure done, pid=77 2024-11-16T09:59:17,521 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=77, resume processing ppid=76 2024-11-16T09:59:17,521 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=77, ppid=76, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 563 msec 2024-11-16T09:59:17,523 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=76, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=76, table=TestAcidGuarantees in 569 msec 2024-11-16T09:59:17,583 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37877 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=76 2024-11-16T09:59:17,583 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-16T09:59:17,584 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37877 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-16T09:59:17,585 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37877 {}] procedure2.ProcedureExecutor(1139): Stored pid=78, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=78, table=TestAcidGuarantees 2024-11-16T09:59:17,586 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=78, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=78, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-16T09:59:17,586 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37877 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=78 2024-11-16T09:59:17,586 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=78, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=78, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-16T09:59:17,587 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=79, ppid=78, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-16T09:59:17,693 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37877 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=78 2024-11-16T09:59:17,739 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=37267 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=79 2024-11-16T09:59:17,739 DEBUG [RS_FLUSH_OPERATIONS-regionserver/037c58a19ddb:0-1 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731751137553.f3ed23ac403772e0fe4ca9ee892544ec. 2024-11-16T09:59:17,739 DEBUG [RS_FLUSH_OPERATIONS-regionserver/037c58a19ddb:0-1 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.HRegion(2603): Flush status journal for f3ed23ac403772e0fe4ca9ee892544ec: 2024-11-16T09:59:17,739 DEBUG [RS_FLUSH_OPERATIONS-regionserver/037c58a19ddb:0-1 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731751137553.f3ed23ac403772e0fe4ca9ee892544ec. 2024-11-16T09:59:17,739 DEBUG [RS_FLUSH_OPERATIONS-regionserver/037c58a19ddb:0-1 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=79 2024-11-16T09:59:17,740 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37877 {}] master.HMaster(4169): Remote procedure done, pid=79 2024-11-16T09:59:17,743 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=79, resume processing ppid=78 2024-11-16T09:59:17,743 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=79, ppid=78, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 154 msec 2024-11-16T09:59:17,745 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=78, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=78, table=TestAcidGuarantees in 160 msec 2024-11-16T09:59:17,875 DEBUG [RS:0;037c58a19ddb:37267-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/f3ed23ac403772e0fe4ca9ee892544ec/.tmp/B/921ee41b9b074ed2b0d2e2c28b2edec3 as hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/f3ed23ac403772e0fe4ca9ee892544ec/B/921ee41b9b074ed2b0d2e2c28b2edec3 2024-11-16T09:59:17,881 INFO [RS:0;037c58a19ddb:37267-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in f3ed23ac403772e0fe4ca9ee892544ec/B of f3ed23ac403772e0fe4ca9ee892544ec into 921ee41b9b074ed2b0d2e2c28b2edec3(size=12.8 K), total size for store is 24.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-16T09:59:17,881 DEBUG [RS:0;037c58a19ddb:37267-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for f3ed23ac403772e0fe4ca9ee892544ec: 2024-11-16T09:59:17,881 INFO [RS:0;037c58a19ddb:37267-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731751137553.f3ed23ac403772e0fe4ca9ee892544ec., storeName=f3ed23ac403772e0fe4ca9ee892544ec/B, priority=13, startTime=1731751157413; duration=0sec 2024-11-16T09:59:17,881 DEBUG [RS:0;037c58a19ddb:37267-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-16T09:59:17,881 DEBUG [RS:0;037c58a19ddb:37267-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: f3ed23ac403772e0fe4ca9ee892544ec:B 2024-11-16T09:59:17,881 DEBUG [RS:0;037c58a19ddb:37267-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-16T09:59:17,883 DEBUG [RS:0;037c58a19ddb:37267-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 49886 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-16T09:59:17,883 DEBUG [RS:0;037c58a19ddb:37267-shortCompactions-0 {}] regionserver.HStore(1541): f3ed23ac403772e0fe4ca9ee892544ec/C is initiating minor compaction (all files) 2024-11-16T09:59:17,883 INFO [RS:0;037c58a19ddb:37267-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of f3ed23ac403772e0fe4ca9ee892544ec/C in TestAcidGuarantees,,1731751137553.f3ed23ac403772e0fe4ca9ee892544ec. 2024-11-16T09:59:17,883 DEBUG [RS:0;037c58a19ddb:37267-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/f3ed23ac403772e0fe4ca9ee892544ec/.tmp/A/b097f1e4754c45e7863be4f0cb423c62 as hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/f3ed23ac403772e0fe4ca9ee892544ec/A/b097f1e4754c45e7863be4f0cb423c62 2024-11-16T09:59:17,883 INFO [RS:0;037c58a19ddb:37267-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/f3ed23ac403772e0fe4ca9ee892544ec/C/42f37ce77c114494ab1505a0d320671a, hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/f3ed23ac403772e0fe4ca9ee892544ec/C/9b15a7dac9a24c048f9b79c39fbc2aad, hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/f3ed23ac403772e0fe4ca9ee892544ec/C/24ccf25c8a5a4d6e8f0d6ebdf95b6954, hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/f3ed23ac403772e0fe4ca9ee892544ec/C/76b2417e63f04287845faacef82c7350] into tmpdir=hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/f3ed23ac403772e0fe4ca9ee892544ec/.tmp, totalSize=48.7 K 2024-11-16T09:59:17,883 DEBUG [RS:0;037c58a19ddb:37267-shortCompactions-0 {}] compactions.Compactor(225): Compacting 42f37ce77c114494ab1505a0d320671a, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=290, earliestPutTs=1731751153227 2024-11-16T09:59:17,883 DEBUG [RS:0;037c58a19ddb:37267-shortCompactions-0 {}] compactions.Compactor(225): Compacting 9b15a7dac9a24c048f9b79c39fbc2aad, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=313, earliestPutTs=1731751153879 2024-11-16T09:59:17,884 DEBUG [RS:0;037c58a19ddb:37267-shortCompactions-0 {}] compactions.Compactor(225): Compacting 24ccf25c8a5a4d6e8f0d6ebdf95b6954, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=330, earliestPutTs=1731751156045 2024-11-16T09:59:17,884 DEBUG [RS:0;037c58a19ddb:37267-shortCompactions-0 {}] compactions.Compactor(225): Compacting 76b2417e63f04287845faacef82c7350, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=352, earliestPutTs=1731751156184 2024-11-16T09:59:17,888 INFO [RS:0;037c58a19ddb:37267-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in f3ed23ac403772e0fe4ca9ee892544ec/A of f3ed23ac403772e0fe4ca9ee892544ec into b097f1e4754c45e7863be4f0cb423c62(size=12.8 K), total size for store is 24.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-16T09:59:17,888 DEBUG [RS:0;037c58a19ddb:37267-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for f3ed23ac403772e0fe4ca9ee892544ec: 2024-11-16T09:59:17,888 INFO [RS:0;037c58a19ddb:37267-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731751137553.f3ed23ac403772e0fe4ca9ee892544ec., storeName=f3ed23ac403772e0fe4ca9ee892544ec/A, priority=13, startTime=1731751157413; duration=0sec 2024-11-16T09:59:17,888 DEBUG [RS:0;037c58a19ddb:37267-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-16T09:59:17,888 DEBUG [RS:0;037c58a19ddb:37267-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: f3ed23ac403772e0fe4ca9ee892544ec:A 2024-11-16T09:59:17,895 INFO [RS:0;037c58a19ddb:37267-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): f3ed23ac403772e0fe4ca9ee892544ec#C#compaction#284 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-16T09:59:17,896 DEBUG [RS:0;037c58a19ddb:37267-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/f3ed23ac403772e0fe4ca9ee892544ec/.tmp/C/cec8fedd185343628fdb34006411b43d is 50, key is test_row_0/C:col10/1731751156184/Put/seqid=0 2024-11-16T09:59:17,903 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37877 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=78 2024-11-16T09:59:17,903 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-16T09:59:17,903 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37877 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-16T09:59:17,904 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37877 {}] procedure2.ProcedureExecutor(1139): Stored pid=80, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=80, table=TestAcidGuarantees 2024-11-16T09:59:17,905 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37877 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=80 2024-11-16T09:59:17,906 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=80, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=80, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-16T09:59:17,907 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=80, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=80, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-16T09:59:17,907 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=81, ppid=80, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-16T09:59:17,920 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37591 is added to blk_1073742153_1329 (size=13119) 2024-11-16T09:59:18,012 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37877 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=80 2024-11-16T09:59:18,059 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=37267 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=81 2024-11-16T09:59:18,060 DEBUG [RS_FLUSH_OPERATIONS-regionserver/037c58a19ddb:0-2 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731751137553.f3ed23ac403772e0fe4ca9ee892544ec. 2024-11-16T09:59:18,060 DEBUG [RS_FLUSH_OPERATIONS-regionserver/037c58a19ddb:0-2 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.HRegion(2603): Flush status journal for f3ed23ac403772e0fe4ca9ee892544ec: 2024-11-16T09:59:18,060 DEBUG [RS_FLUSH_OPERATIONS-regionserver/037c58a19ddb:0-2 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731751137553.f3ed23ac403772e0fe4ca9ee892544ec. 2024-11-16T09:59:18,060 DEBUG [RS_FLUSH_OPERATIONS-regionserver/037c58a19ddb:0-2 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=81 2024-11-16T09:59:18,061 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37877 {}] master.HMaster(4169): Remote procedure done, pid=81 2024-11-16T09:59:18,064 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=81, resume processing ppid=80 2024-11-16T09:59:18,064 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=81, ppid=80, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 155 msec 2024-11-16T09:59:18,066 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=80, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=80, table=TestAcidGuarantees in 162 msec 2024-11-16T09:59:18,085 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] regionserver.HRegion(8855): Flush requested on f3ed23ac403772e0fe4ca9ee892544ec 2024-11-16T09:59:18,085 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-16T09:59:18,085 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing f3ed23ac403772e0fe4ca9ee892544ec 3/3 column families, dataSize=53.67 KB heapSize=141.38 KB 2024-11-16T09:59:18,090 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/f3ed23ac403772e0fe4ca9ee892544ec/.tmp/A/1e115d5e923f4de2a49c0fc719d77e9c is 50, key is test_row_0/A:col10/1731751158079/Put/seqid=0 2024-11-16T09:59:18,095 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37591 is added to blk_1073742154_1330 (size=12301) 2024-11-16T09:59:18,130 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3ed23ac403772e0fe4ca9ee892544ec, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T09:59:18,131 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] ipc.CallRunner(138): callId: 119 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34408 deadline: 1731751218127, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3ed23ac403772e0fe4ca9ee892544ec, server=037c58a19ddb,37267,1731751077844 2024-11-16T09:59:18,131 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731751137553.f3ed23ac403772e0fe4ca9ee892544ec., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 , the old value is region=TestAcidGuarantees,,1731751137553.f3ed23ac403772e0fe4ca9ee892544ec., hostname=037c58a19ddb,37267,1731751077844, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3ed23ac403772e0fe4ca9ee892544ec, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:59:18,131 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731751137553.f3ed23ac403772e0fe4ca9ee892544ec., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3ed23ac403772e0fe4ca9ee892544ec, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:59:18,131 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731751137553.f3ed23ac403772e0fe4ca9ee892544ec., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 because the exception is null or not the one we care about 2024-11-16T09:59:18,132 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3ed23ac403772e0fe4ca9ee892544ec, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T09:59:18,132 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] ipc.CallRunner(138): callId: 117 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34414 deadline: 1731751218129, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3ed23ac403772e0fe4ca9ee892544ec, server=037c58a19ddb,37267,1731751077844 2024-11-16T09:59:18,133 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731751137553.f3ed23ac403772e0fe4ca9ee892544ec., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 , the old value is region=TestAcidGuarantees,,1731751137553.f3ed23ac403772e0fe4ca9ee892544ec., hostname=037c58a19ddb,37267,1731751077844, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3ed23ac403772e0fe4ca9ee892544ec, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:59:18,133 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731751137553.f3ed23ac403772e0fe4ca9ee892544ec., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3ed23ac403772e0fe4ca9ee892544ec, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:59:18,133 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731751137553.f3ed23ac403772e0fe4ca9ee892544ec., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 because the exception is null or not the one we care about 2024-11-16T09:59:18,223 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37877 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=80 2024-11-16T09:59:18,223 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-16T09:59:18,224 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37877 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-16T09:59:18,225 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37877 {}] procedure2.ProcedureExecutor(1139): Stored pid=82, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=82, table=TestAcidGuarantees 2024-11-16T09:59:18,226 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37877 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=82 2024-11-16T09:59:18,226 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=82, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=82, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-16T09:59:18,227 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=82, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=82, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-16T09:59:18,227 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=83, ppid=82, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-16T09:59:18,235 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3ed23ac403772e0fe4ca9ee892544ec, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T09:59:18,235 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] ipc.CallRunner(138): callId: 120 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34408 deadline: 1731751218232, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3ed23ac403772e0fe4ca9ee892544ec, server=037c58a19ddb,37267,1731751077844 2024-11-16T09:59:18,235 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731751137553.f3ed23ac403772e0fe4ca9ee892544ec., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 , the old value is region=TestAcidGuarantees,,1731751137553.f3ed23ac403772e0fe4ca9ee892544ec., hostname=037c58a19ddb,37267,1731751077844, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3ed23ac403772e0fe4ca9ee892544ec, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:59:18,235 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731751137553.f3ed23ac403772e0fe4ca9ee892544ec., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3ed23ac403772e0fe4ca9ee892544ec, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:59:18,235 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731751137553.f3ed23ac403772e0fe4ca9ee892544ec., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 because the exception is null or not the one we care about 2024-11-16T09:59:18,245 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3ed23ac403772e0fe4ca9ee892544ec, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T09:59:18,245 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] ipc.CallRunner(138): callId: 118 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34414 deadline: 1731751218242, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3ed23ac403772e0fe4ca9ee892544ec, server=037c58a19ddb,37267,1731751077844 2024-11-16T09:59:18,245 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731751137553.f3ed23ac403772e0fe4ca9ee892544ec., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 , the old value is region=TestAcidGuarantees,,1731751137553.f3ed23ac403772e0fe4ca9ee892544ec., hostname=037c58a19ddb,37267,1731751077844, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3ed23ac403772e0fe4ca9ee892544ec, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:59:18,245 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731751137553.f3ed23ac403772e0fe4ca9ee892544ec., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3ed23ac403772e0fe4ca9ee892544ec, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:59:18,245 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731751137553.f3ed23ac403772e0fe4ca9ee892544ec., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 because the exception is null or not the one we care about 2024-11-16T09:59:18,326 DEBUG [RS:0;037c58a19ddb:37267-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/f3ed23ac403772e0fe4ca9ee892544ec/.tmp/C/cec8fedd185343628fdb34006411b43d as hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/f3ed23ac403772e0fe4ca9ee892544ec/C/cec8fedd185343628fdb34006411b43d 2024-11-16T09:59:18,330 INFO [RS:0;037c58a19ddb:37267-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in f3ed23ac403772e0fe4ca9ee892544ec/C of f3ed23ac403772e0fe4ca9ee892544ec into cec8fedd185343628fdb34006411b43d(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-16T09:59:18,330 DEBUG [RS:0;037c58a19ddb:37267-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for f3ed23ac403772e0fe4ca9ee892544ec: 2024-11-16T09:59:18,330 INFO [RS:0;037c58a19ddb:37267-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731751137553.f3ed23ac403772e0fe4ca9ee892544ec., storeName=f3ed23ac403772e0fe4ca9ee892544ec/C, priority=12, startTime=1731751157413; duration=0sec 2024-11-16T09:59:18,330 DEBUG [RS:0;037c58a19ddb:37267-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-16T09:59:18,330 DEBUG [RS:0;037c58a19ddb:37267-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: f3ed23ac403772e0fe4ca9ee892544ec:C 2024-11-16T09:59:18,332 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37877 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=82 2024-11-16T09:59:18,345 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3ed23ac403772e0fe4ca9ee892544ec, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T09:59:18,345 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] ipc.CallRunner(138): callId: 123 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34368 deadline: 1731751218343, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3ed23ac403772e0fe4ca9ee892544ec, server=037c58a19ddb,37267,1731751077844 2024-11-16T09:59:18,345 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3ed23ac403772e0fe4ca9ee892544ec, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T09:59:18,346 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] ipc.CallRunner(138): callId: 118 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34386 deadline: 1731751218343, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3ed23ac403772e0fe4ca9ee892544ec, server=037c58a19ddb,37267,1731751077844 2024-11-16T09:59:18,346 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3ed23ac403772e0fe4ca9ee892544ec, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T09:59:18,346 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] ipc.CallRunner(138): callId: 112 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34382 deadline: 1731751218343, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3ed23ac403772e0fe4ca9ee892544ec, server=037c58a19ddb,37267,1731751077844 2024-11-16T09:59:18,345 WARN [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRpcRetryingCaller(177): Call to 037c58a19ddb,37267,1731751077844 for 'test_row_0' in f3ed23ac403772e0fe4ca9ee892544ec of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2155 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3ed23ac403772e0fe4ca9ee892544ec, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor44.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3ed23ac403772e0fe4ca9ee892544ec, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-16T09:59:18,346 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731751137553.f3ed23ac403772e0fe4ca9ee892544ec., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 , the old value is region=TestAcidGuarantees,,1731751137553.f3ed23ac403772e0fe4ca9ee892544ec., hostname=037c58a19ddb,37267,1731751077844, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3ed23ac403772e0fe4ca9ee892544ec, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:59:18,346 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731751137553.f3ed23ac403772e0fe4ca9ee892544ec., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3ed23ac403772e0fe4ca9ee892544ec, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:59:18,346 WARN [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRpcRetryingCaller(177): Call to 037c58a19ddb,37267,1731751077844 for 'test_row_2' in f3ed23ac403772e0fe4ca9ee892544ec of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2154 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3ed23ac403772e0fe4ca9ee892544ec, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor44.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3ed23ac403772e0fe4ca9ee892544ec, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-16T09:59:18,346 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731751137553.f3ed23ac403772e0fe4ca9ee892544ec., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 because the exception is null or not the one we care about 2024-11-16T09:59:18,346 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731751137553.f3ed23ac403772e0fe4ca9ee892544ec., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 , the old value is region=TestAcidGuarantees,,1731751137553.f3ed23ac403772e0fe4ca9ee892544ec., hostname=037c58a19ddb,37267,1731751077844, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3ed23ac403772e0fe4ca9ee892544ec, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:59:18,346 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731751137553.f3ed23ac403772e0fe4ca9ee892544ec., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3ed23ac403772e0fe4ca9ee892544ec, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:59:18,346 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731751137553.f3ed23ac403772e0fe4ca9ee892544ec., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 because the exception is null or not the one we care about 2024-11-16T09:59:18,346 WARN [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRpcRetryingCaller(177): Call to 037c58a19ddb,37267,1731751077844 for 'test_row_2' in f3ed23ac403772e0fe4ca9ee892544ec of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2157 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3ed23ac403772e0fe4ca9ee892544ec, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor44.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3ed23ac403772e0fe4ca9ee892544ec, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-16T09:59:18,347 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731751137553.f3ed23ac403772e0fe4ca9ee892544ec., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 , the old value is region=TestAcidGuarantees,,1731751137553.f3ed23ac403772e0fe4ca9ee892544ec., hostname=037c58a19ddb,37267,1731751077844, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3ed23ac403772e0fe4ca9ee892544ec, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:59:18,347 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731751137553.f3ed23ac403772e0fe4ca9ee892544ec., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3ed23ac403772e0fe4ca9ee892544ec, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:59:18,347 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731751137553.f3ed23ac403772e0fe4ca9ee892544ec., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 because the exception is null or not the one we care about 2024-11-16T09:59:18,379 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=37267 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=83 2024-11-16T09:59:18,379 DEBUG [RS_FLUSH_OPERATIONS-regionserver/037c58a19ddb:0-0 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731751137553.f3ed23ac403772e0fe4ca9ee892544ec. 2024-11-16T09:59:18,379 DEBUG [RS_FLUSH_OPERATIONS-regionserver/037c58a19ddb:0-0 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731751137553.f3ed23ac403772e0fe4ca9ee892544ec. as already flushing 2024-11-16T09:59:18,379 DEBUG [RS_FLUSH_OPERATIONS-regionserver/037c58a19ddb:0-0 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731751137553.f3ed23ac403772e0fe4ca9ee892544ec. 2024-11-16T09:59:18,379 ERROR [RS_FLUSH_OPERATIONS-regionserver/037c58a19ddb:0-0 {event_type=RS_FLUSH_REGIONS, pid=83}] handler.RSProcedureHandler(58): pid=83 java.io.IOException: Unable to complete flush {ENCODED => f3ed23ac403772e0fe4ca9ee892544ec, NAME => 'TestAcidGuarantees,,1731751137553.f3ed23ac403772e0fe4ca9ee892544ec.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T09:59:18,380 DEBUG [RS_FLUSH_OPERATIONS-regionserver/037c58a19ddb:0-0 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=83 java.io.IOException: Unable to complete flush {ENCODED => f3ed23ac403772e0fe4ca9ee892544ec, NAME => 'TestAcidGuarantees,,1731751137553.f3ed23ac403772e0fe4ca9ee892544ec.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T09:59:18,380 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37877 {}] master.HMaster(4177): Remote procedure failed, pid=83 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => f3ed23ac403772e0fe4ca9ee892544ec, NAME => 'TestAcidGuarantees,,1731751137553.f3ed23ac403772e0fe4ca9ee892544ec.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => f3ed23ac403772e0fe4ca9ee892544ec, NAME => 'TestAcidGuarantees,,1731751137553.f3ed23ac403772e0fe4ca9ee892544ec.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T09:59:18,445 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3ed23ac403772e0fe4ca9ee892544ec, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T09:59:18,445 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] ipc.CallRunner(138): callId: 121 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34408 deadline: 1731751218443, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3ed23ac403772e0fe4ca9ee892544ec, server=037c58a19ddb,37267,1731751077844 2024-11-16T09:59:18,445 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731751137553.f3ed23ac403772e0fe4ca9ee892544ec., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 , the old value is region=TestAcidGuarantees,,1731751137553.f3ed23ac403772e0fe4ca9ee892544ec., hostname=037c58a19ddb,37267,1731751077844, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3ed23ac403772e0fe4ca9ee892544ec, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:59:18,445 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731751137553.f3ed23ac403772e0fe4ca9ee892544ec., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3ed23ac403772e0fe4ca9ee892544ec, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:59:18,445 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731751137553.f3ed23ac403772e0fe4ca9ee892544ec., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 because the exception is null or not the one we care about 2024-11-16T09:59:18,453 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3ed23ac403772e0fe4ca9ee892544ec, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T09:59:18,453 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] ipc.CallRunner(138): callId: 119 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34414 deadline: 1731751218453, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3ed23ac403772e0fe4ca9ee892544ec, server=037c58a19ddb,37267,1731751077844 2024-11-16T09:59:18,454 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731751137553.f3ed23ac403772e0fe4ca9ee892544ec., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 , the old value is region=TestAcidGuarantees,,1731751137553.f3ed23ac403772e0fe4ca9ee892544ec., hostname=037c58a19ddb,37267,1731751077844, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3ed23ac403772e0fe4ca9ee892544ec, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:59:18,454 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731751137553.f3ed23ac403772e0fe4ca9ee892544ec., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3ed23ac403772e0fe4ca9ee892544ec, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:59:18,454 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731751137553.f3ed23ac403772e0fe4ca9ee892544ec., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 because the exception is null or not the one we care about 2024-11-16T09:59:18,496 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=367 (bloomFilter=true), to=hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/f3ed23ac403772e0fe4ca9ee892544ec/.tmp/A/1e115d5e923f4de2a49c0fc719d77e9c 2024-11-16T09:59:18,503 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/f3ed23ac403772e0fe4ca9ee892544ec/.tmp/B/9020014cebb84c3d92e7decc88f8a37e is 50, key is test_row_0/B:col10/1731751158079/Put/seqid=0 2024-11-16T09:59:18,507 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37591 is added to blk_1073742155_1331 (size=12301) 2024-11-16T09:59:18,507 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=367 (bloomFilter=true), to=hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/f3ed23ac403772e0fe4ca9ee892544ec/.tmp/B/9020014cebb84c3d92e7decc88f8a37e 2024-11-16T09:59:18,516 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/f3ed23ac403772e0fe4ca9ee892544ec/.tmp/C/5c78ce09ea8c45479341bc9aeaa9aece is 50, key is test_row_0/C:col10/1731751158079/Put/seqid=0 2024-11-16T09:59:18,519 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37591 is added to blk_1073742156_1332 (size=12301) 2024-11-16T09:59:18,532 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=37267 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=83 2024-11-16T09:59:18,532 DEBUG [RS_FLUSH_OPERATIONS-regionserver/037c58a19ddb:0-1 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731751137553.f3ed23ac403772e0fe4ca9ee892544ec. 2024-11-16T09:59:18,532 DEBUG [RS_FLUSH_OPERATIONS-regionserver/037c58a19ddb:0-1 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731751137553.f3ed23ac403772e0fe4ca9ee892544ec. as already flushing 2024-11-16T09:59:18,532 DEBUG [RS_FLUSH_OPERATIONS-regionserver/037c58a19ddb:0-1 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731751137553.f3ed23ac403772e0fe4ca9ee892544ec. 2024-11-16T09:59:18,533 ERROR [RS_FLUSH_OPERATIONS-regionserver/037c58a19ddb:0-1 {event_type=RS_FLUSH_REGIONS, pid=83}] handler.RSProcedureHandler(58): pid=83 java.io.IOException: Unable to complete flush {ENCODED => f3ed23ac403772e0fe4ca9ee892544ec, NAME => 'TestAcidGuarantees,,1731751137553.f3ed23ac403772e0fe4ca9ee892544ec.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T09:59:18,533 DEBUG [RS_FLUSH_OPERATIONS-regionserver/037c58a19ddb:0-1 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=83 java.io.IOException: Unable to complete flush {ENCODED => f3ed23ac403772e0fe4ca9ee892544ec, NAME => 'TestAcidGuarantees,,1731751137553.f3ed23ac403772e0fe4ca9ee892544ec.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T09:59:18,533 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37877 {}] master.HMaster(4177): Remote procedure failed, pid=83 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => f3ed23ac403772e0fe4ca9ee892544ec, NAME => 'TestAcidGuarantees,,1731751137553.f3ed23ac403772e0fe4ca9ee892544ec.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => f3ed23ac403772e0fe4ca9ee892544ec, NAME => 'TestAcidGuarantees,,1731751137553.f3ed23ac403772e0fe4ca9ee892544ec.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T09:59:18,543 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37877 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=82 2024-11-16T09:59:18,685 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=37267 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=83 2024-11-16T09:59:18,685 DEBUG [RS_FLUSH_OPERATIONS-regionserver/037c58a19ddb:0-2 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731751137553.f3ed23ac403772e0fe4ca9ee892544ec. 2024-11-16T09:59:18,685 DEBUG [RS_FLUSH_OPERATIONS-regionserver/037c58a19ddb:0-2 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731751137553.f3ed23ac403772e0fe4ca9ee892544ec. as already flushing 2024-11-16T09:59:18,685 DEBUG [RS_FLUSH_OPERATIONS-regionserver/037c58a19ddb:0-2 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731751137553.f3ed23ac403772e0fe4ca9ee892544ec. 2024-11-16T09:59:18,686 ERROR [RS_FLUSH_OPERATIONS-regionserver/037c58a19ddb:0-2 {event_type=RS_FLUSH_REGIONS, pid=83}] handler.RSProcedureHandler(58): pid=83 java.io.IOException: Unable to complete flush {ENCODED => f3ed23ac403772e0fe4ca9ee892544ec, NAME => 'TestAcidGuarantees,,1731751137553.f3ed23ac403772e0fe4ca9ee892544ec.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T09:59:18,686 DEBUG [RS_FLUSH_OPERATIONS-regionserver/037c58a19ddb:0-2 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=83 java.io.IOException: Unable to complete flush {ENCODED => f3ed23ac403772e0fe4ca9ee892544ec, NAME => 'TestAcidGuarantees,,1731751137553.f3ed23ac403772e0fe4ca9ee892544ec.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T09:59:18,686 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37877 {}] master.HMaster(4177): Remote procedure failed, pid=83 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => f3ed23ac403772e0fe4ca9ee892544ec, NAME => 'TestAcidGuarantees,,1731751137553.f3ed23ac403772e0fe4ca9ee892544ec.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => f3ed23ac403772e0fe4ca9ee892544ec, NAME => 'TestAcidGuarantees,,1731751137553.f3ed23ac403772e0fe4ca9ee892544ec.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T09:59:18,755 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3ed23ac403772e0fe4ca9ee892544ec, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T09:59:18,755 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] ipc.CallRunner(138): callId: 122 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34408 deadline: 1731751218753, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3ed23ac403772e0fe4ca9ee892544ec, server=037c58a19ddb,37267,1731751077844 2024-11-16T09:59:18,755 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731751137553.f3ed23ac403772e0fe4ca9ee892544ec., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 , the old value is region=TestAcidGuarantees,,1731751137553.f3ed23ac403772e0fe4ca9ee892544ec., hostname=037c58a19ddb,37267,1731751077844, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3ed23ac403772e0fe4ca9ee892544ec, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:59:18,755 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731751137553.f3ed23ac403772e0fe4ca9ee892544ec., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3ed23ac403772e0fe4ca9ee892544ec, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:59:18,755 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731751137553.f3ed23ac403772e0fe4ca9ee892544ec., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 because the exception is null or not the one we care about 2024-11-16T09:59:18,764 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3ed23ac403772e0fe4ca9ee892544ec, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T09:59:18,764 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] ipc.CallRunner(138): callId: 120 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34414 deadline: 1731751218762, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3ed23ac403772e0fe4ca9ee892544ec, server=037c58a19ddb,37267,1731751077844 2024-11-16T09:59:18,764 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731751137553.f3ed23ac403772e0fe4ca9ee892544ec., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 , the old value is region=TestAcidGuarantees,,1731751137553.f3ed23ac403772e0fe4ca9ee892544ec., hostname=037c58a19ddb,37267,1731751077844, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3ed23ac403772e0fe4ca9ee892544ec, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:59:18,764 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731751137553.f3ed23ac403772e0fe4ca9ee892544ec., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3ed23ac403772e0fe4ca9ee892544ec, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:59:18,764 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731751137553.f3ed23ac403772e0fe4ca9ee892544ec., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 because the exception is null or not the one we care about 2024-11-16T09:59:18,838 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=37267 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=83 2024-11-16T09:59:18,838 DEBUG [RS_FLUSH_OPERATIONS-regionserver/037c58a19ddb:0-0 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731751137553.f3ed23ac403772e0fe4ca9ee892544ec. 2024-11-16T09:59:18,838 DEBUG [RS_FLUSH_OPERATIONS-regionserver/037c58a19ddb:0-0 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731751137553.f3ed23ac403772e0fe4ca9ee892544ec. as already flushing 2024-11-16T09:59:18,838 DEBUG [RS_FLUSH_OPERATIONS-regionserver/037c58a19ddb:0-0 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731751137553.f3ed23ac403772e0fe4ca9ee892544ec. 2024-11-16T09:59:18,838 ERROR [RS_FLUSH_OPERATIONS-regionserver/037c58a19ddb:0-0 {event_type=RS_FLUSH_REGIONS, pid=83}] handler.RSProcedureHandler(58): pid=83 java.io.IOException: Unable to complete flush {ENCODED => f3ed23ac403772e0fe4ca9ee892544ec, NAME => 'TestAcidGuarantees,,1731751137553.f3ed23ac403772e0fe4ca9ee892544ec.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T09:59:18,839 DEBUG [RS_FLUSH_OPERATIONS-regionserver/037c58a19ddb:0-0 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=83 java.io.IOException: Unable to complete flush {ENCODED => f3ed23ac403772e0fe4ca9ee892544ec, NAME => 'TestAcidGuarantees,,1731751137553.f3ed23ac403772e0fe4ca9ee892544ec.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T09:59:18,839 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37877 {}] master.HMaster(4177): Remote procedure failed, pid=83 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => f3ed23ac403772e0fe4ca9ee892544ec, NAME => 'TestAcidGuarantees,,1731751137553.f3ed23ac403772e0fe4ca9ee892544ec.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => f3ed23ac403772e0fe4ca9ee892544ec, NAME => 'TestAcidGuarantees,,1731751137553.f3ed23ac403772e0fe4ca9ee892544ec.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T09:59:18,853 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37877 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=82 2024-11-16T09:59:18,920 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=367 (bloomFilter=true), to=hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/f3ed23ac403772e0fe4ca9ee892544ec/.tmp/C/5c78ce09ea8c45479341bc9aeaa9aece 2024-11-16T09:59:18,925 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/f3ed23ac403772e0fe4ca9ee892544ec/.tmp/A/1e115d5e923f4de2a49c0fc719d77e9c as hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/f3ed23ac403772e0fe4ca9ee892544ec/A/1e115d5e923f4de2a49c0fc719d77e9c 2024-11-16T09:59:18,929 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/f3ed23ac403772e0fe4ca9ee892544ec/A/1e115d5e923f4de2a49c0fc719d77e9c, entries=150, sequenceid=367, filesize=12.0 K 2024-11-16T09:59:18,930 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/f3ed23ac403772e0fe4ca9ee892544ec/.tmp/B/9020014cebb84c3d92e7decc88f8a37e as hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/f3ed23ac403772e0fe4ca9ee892544ec/B/9020014cebb84c3d92e7decc88f8a37e 2024-11-16T09:59:18,933 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/f3ed23ac403772e0fe4ca9ee892544ec/B/9020014cebb84c3d92e7decc88f8a37e, entries=150, sequenceid=367, filesize=12.0 K 2024-11-16T09:59:18,934 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/f3ed23ac403772e0fe4ca9ee892544ec/.tmp/C/5c78ce09ea8c45479341bc9aeaa9aece as hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/f3ed23ac403772e0fe4ca9ee892544ec/C/5c78ce09ea8c45479341bc9aeaa9aece 2024-11-16T09:59:18,938 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/f3ed23ac403772e0fe4ca9ee892544ec/C/5c78ce09ea8c45479341bc9aeaa9aece, entries=150, sequenceid=367, filesize=12.0 K 2024-11-16T09:59:18,939 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~141.33 KB/144720, currentSize=147.60 KB/151140 for f3ed23ac403772e0fe4ca9ee892544ec in 854ms, sequenceid=367, compaction requested=true 2024-11-16T09:59:18,939 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for f3ed23ac403772e0fe4ca9ee892544ec: 2024-11-16T09:59:18,939 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store f3ed23ac403772e0fe4ca9ee892544ec:A, priority=-2147483648, current under compaction store size is 1 2024-11-16T09:59:18,939 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-16T09:59:18,939 DEBUG [RS:0;037c58a19ddb:37267-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-16T09:59:18,939 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store f3ed23ac403772e0fe4ca9ee892544ec:B, priority=-2147483648, current under compaction store size is 2 2024-11-16T09:59:18,939 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-16T09:59:18,939 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store f3ed23ac403772e0fe4ca9ee892544ec:C, priority=-2147483648, current under compaction store size is 3 2024-11-16T09:59:18,939 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-16T09:59:18,939 DEBUG [RS:0;037c58a19ddb:37267-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-16T09:59:18,940 DEBUG [RS:0;037c58a19ddb:37267-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37687 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-16T09:59:18,940 DEBUG [RS:0;037c58a19ddb:37267-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37687 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-16T09:59:18,940 DEBUG [RS:0;037c58a19ddb:37267-longCompactions-0 {}] regionserver.HStore(1541): f3ed23ac403772e0fe4ca9ee892544ec/B is initiating minor compaction (all files) 2024-11-16T09:59:18,940 DEBUG [RS:0;037c58a19ddb:37267-shortCompactions-0 {}] regionserver.HStore(1541): f3ed23ac403772e0fe4ca9ee892544ec/A is initiating minor compaction (all files) 2024-11-16T09:59:18,940 INFO [RS:0;037c58a19ddb:37267-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of f3ed23ac403772e0fe4ca9ee892544ec/B in TestAcidGuarantees,,1731751137553.f3ed23ac403772e0fe4ca9ee892544ec. 2024-11-16T09:59:18,940 INFO [RS:0;037c58a19ddb:37267-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of f3ed23ac403772e0fe4ca9ee892544ec/A in TestAcidGuarantees,,1731751137553.f3ed23ac403772e0fe4ca9ee892544ec. 2024-11-16T09:59:18,940 INFO [RS:0;037c58a19ddb:37267-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/f3ed23ac403772e0fe4ca9ee892544ec/B/921ee41b9b074ed2b0d2e2c28b2edec3, hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/f3ed23ac403772e0fe4ca9ee892544ec/B/7f75e570933e48349567b87f6b6576d1, hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/f3ed23ac403772e0fe4ca9ee892544ec/B/9020014cebb84c3d92e7decc88f8a37e] into tmpdir=hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/f3ed23ac403772e0fe4ca9ee892544ec/.tmp, totalSize=36.8 K 2024-11-16T09:59:18,940 INFO [RS:0;037c58a19ddb:37267-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/f3ed23ac403772e0fe4ca9ee892544ec/A/b097f1e4754c45e7863be4f0cb423c62, hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/f3ed23ac403772e0fe4ca9ee892544ec/A/d07186c35def4c13966a8948333333ac, hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/f3ed23ac403772e0fe4ca9ee892544ec/A/1e115d5e923f4de2a49c0fc719d77e9c] into tmpdir=hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/f3ed23ac403772e0fe4ca9ee892544ec/.tmp, totalSize=36.8 K 2024-11-16T09:59:18,940 DEBUG [RS:0;037c58a19ddb:37267-longCompactions-0 {}] compactions.Compactor(225): Compacting 921ee41b9b074ed2b0d2e2c28b2edec3, keycount=150, bloomtype=ROW, size=12.8 K, encoding=NONE, compression=NONE, seqNum=330, earliestPutTs=1731751156045 2024-11-16T09:59:18,940 DEBUG [RS:0;037c58a19ddb:37267-shortCompactions-0 {}] compactions.Compactor(225): Compacting b097f1e4754c45e7863be4f0cb423c62, keycount=150, bloomtype=ROW, size=12.8 K, encoding=NONE, compression=NONE, seqNum=330, earliestPutTs=1731751156045 2024-11-16T09:59:18,941 DEBUG [RS:0;037c58a19ddb:37267-longCompactions-0 {}] compactions.Compactor(225): Compacting 7f75e570933e48349567b87f6b6576d1, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=352, earliestPutTs=1731751156184 2024-11-16T09:59:18,941 DEBUG [RS:0;037c58a19ddb:37267-shortCompactions-0 {}] compactions.Compactor(225): Compacting d07186c35def4c13966a8948333333ac, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=352, earliestPutTs=1731751156184 2024-11-16T09:59:18,941 DEBUG [RS:0;037c58a19ddb:37267-longCompactions-0 {}] compactions.Compactor(225): Compacting 9020014cebb84c3d92e7decc88f8a37e, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=367, earliestPutTs=1731751158063 2024-11-16T09:59:18,941 DEBUG [RS:0;037c58a19ddb:37267-shortCompactions-0 {}] compactions.Compactor(225): Compacting 1e115d5e923f4de2a49c0fc719d77e9c, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=367, earliestPutTs=1731751158063 2024-11-16T09:59:18,947 INFO [RS:0;037c58a19ddb:37267-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): f3ed23ac403772e0fe4ca9ee892544ec#B#compaction#288 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-16T09:59:18,948 DEBUG [RS:0;037c58a19ddb:37267-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/f3ed23ac403772e0fe4ca9ee892544ec/.tmp/B/651328a01a444b6d81fe1cf63276890b is 50, key is test_row_0/B:col10/1731751158079/Put/seqid=0 2024-11-16T09:59:18,949 INFO [RS:0;037c58a19ddb:37267-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): f3ed23ac403772e0fe4ca9ee892544ec#A#compaction#289 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-16T09:59:18,949 DEBUG [RS:0;037c58a19ddb:37267-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/f3ed23ac403772e0fe4ca9ee892544ec/.tmp/A/a2b10ea75ac040618f8632c5bc0636d4 is 50, key is test_row_0/A:col10/1731751158079/Put/seqid=0 2024-11-16T09:59:18,957 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37591 is added to blk_1073742157_1333 (size=13187) 2024-11-16T09:59:18,962 DEBUG [RS:0;037c58a19ddb:37267-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/f3ed23ac403772e0fe4ca9ee892544ec/.tmp/B/651328a01a444b6d81fe1cf63276890b as hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/f3ed23ac403772e0fe4ca9ee892544ec/B/651328a01a444b6d81fe1cf63276890b 2024-11-16T09:59:18,966 INFO [RS:0;037c58a19ddb:37267-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in f3ed23ac403772e0fe4ca9ee892544ec/B of f3ed23ac403772e0fe4ca9ee892544ec into 651328a01a444b6d81fe1cf63276890b(size=12.9 K), total size for store is 12.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-16T09:59:18,966 DEBUG [RS:0;037c58a19ddb:37267-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for f3ed23ac403772e0fe4ca9ee892544ec: 2024-11-16T09:59:18,966 INFO [RS:0;037c58a19ddb:37267-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731751137553.f3ed23ac403772e0fe4ca9ee892544ec., storeName=f3ed23ac403772e0fe4ca9ee892544ec/B, priority=13, startTime=1731751158939; duration=0sec 2024-11-16T09:59:18,966 DEBUG [RS:0;037c58a19ddb:37267-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-16T09:59:18,966 DEBUG [RS:0;037c58a19ddb:37267-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: f3ed23ac403772e0fe4ca9ee892544ec:B 2024-11-16T09:59:18,966 DEBUG [RS:0;037c58a19ddb:37267-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 2 store files, 0 compacting, 2 eligible, 16 blocking 2024-11-16T09:59:18,967 DEBUG [RS:0;037c58a19ddb:37267-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 0 files of size 0 starting at candidate #-1 after considering 0 permutations with 0 in ratio 2024-11-16T09:59:18,967 DEBUG [RS:0;037c58a19ddb:37267-longCompactions-0 {}] compactions.SortedCompactionPolicy(232): Not compacting files because we only have 0 files ready for compaction. Need 3 to initiate. 2024-11-16T09:59:18,967 DEBUG [RS:0;037c58a19ddb:37267-longCompactions-0 {}] regionserver.CompactSplit(450): Not compacting TestAcidGuarantees,,1731751137553.f3ed23ac403772e0fe4ca9ee892544ec. because compaction request was cancelled 2024-11-16T09:59:18,967 DEBUG [RS:0;037c58a19ddb:37267-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: f3ed23ac403772e0fe4ca9ee892544ec:C 2024-11-16T09:59:18,978 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37591 is added to blk_1073742158_1334 (size=13187) 2024-11-16T09:59:18,991 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=37267 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=83 2024-11-16T09:59:18,991 DEBUG [RS_FLUSH_OPERATIONS-regionserver/037c58a19ddb:0-1 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731751137553.f3ed23ac403772e0fe4ca9ee892544ec. 2024-11-16T09:59:18,992 INFO [RS_FLUSH_OPERATIONS-regionserver/037c58a19ddb:0-1 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.HRegion(2902): Flushing f3ed23ac403772e0fe4ca9ee892544ec 3/3 column families, dataSize=147.60 KB heapSize=387.47 KB 2024-11-16T09:59:18,997 DEBUG [RS_FLUSH_OPERATIONS-regionserver/037c58a19ddb:0-1 {event_type=RS_FLUSH_REGIONS, pid=83}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/f3ed23ac403772e0fe4ca9ee892544ec/.tmp/A/c80e5c0c86dc40869a7ee85491ab01c0 is 50, key is test_row_0/A:col10/1731751158118/Put/seqid=0 2024-11-16T09:59:19,009 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37591 is added to blk_1073742159_1335 (size=12301) 2024-11-16T09:59:19,265 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] regionserver.HRegion(8855): Flush requested on f3ed23ac403772e0fe4ca9ee892544ec 2024-11-16T09:59:19,265 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731751137553.f3ed23ac403772e0fe4ca9ee892544ec. as already flushing 2024-11-16T09:59:19,290 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3ed23ac403772e0fe4ca9ee892544ec, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T09:59:19,290 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] ipc.CallRunner(138): callId: 129 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34408 deadline: 1731751219287, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3ed23ac403772e0fe4ca9ee892544ec, server=037c58a19ddb,37267,1731751077844 2024-11-16T09:59:19,291 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731751137553.f3ed23ac403772e0fe4ca9ee892544ec., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 , the old value is region=TestAcidGuarantees,,1731751137553.f3ed23ac403772e0fe4ca9ee892544ec., hostname=037c58a19ddb,37267,1731751077844, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3ed23ac403772e0fe4ca9ee892544ec, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:59:19,291 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731751137553.f3ed23ac403772e0fe4ca9ee892544ec., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3ed23ac403772e0fe4ca9ee892544ec, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:59:19,291 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731751137553.f3ed23ac403772e0fe4ca9ee892544ec., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 because the exception is null or not the one we care about 2024-11-16T09:59:19,291 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3ed23ac403772e0fe4ca9ee892544ec, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T09:59:19,291 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] ipc.CallRunner(138): callId: 124 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34414 deadline: 1731751219288, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3ed23ac403772e0fe4ca9ee892544ec, server=037c58a19ddb,37267,1731751077844 2024-11-16T09:59:19,291 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731751137553.f3ed23ac403772e0fe4ca9ee892544ec., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 , the old value is region=TestAcidGuarantees,,1731751137553.f3ed23ac403772e0fe4ca9ee892544ec., hostname=037c58a19ddb,37267,1731751077844, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3ed23ac403772e0fe4ca9ee892544ec, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:59:19,291 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731751137553.f3ed23ac403772e0fe4ca9ee892544ec., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3ed23ac403772e0fe4ca9ee892544ec, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:59:19,291 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731751137553.f3ed23ac403772e0fe4ca9ee892544ec., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 because the exception is null or not the one we care about 2024-11-16T09:59:19,362 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37877 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=82 2024-11-16T09:59:19,384 DEBUG [RS:0;037c58a19ddb:37267-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/f3ed23ac403772e0fe4ca9ee892544ec/.tmp/A/a2b10ea75ac040618f8632c5bc0636d4 as hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/f3ed23ac403772e0fe4ca9ee892544ec/A/a2b10ea75ac040618f8632c5bc0636d4 2024-11-16T09:59:19,389 INFO [RS:0;037c58a19ddb:37267-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in f3ed23ac403772e0fe4ca9ee892544ec/A of f3ed23ac403772e0fe4ca9ee892544ec into a2b10ea75ac040618f8632c5bc0636d4(size=12.9 K), total size for store is 12.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-16T09:59:19,389 DEBUG [RS:0;037c58a19ddb:37267-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for f3ed23ac403772e0fe4ca9ee892544ec: 2024-11-16T09:59:19,389 INFO [RS:0;037c58a19ddb:37267-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731751137553.f3ed23ac403772e0fe4ca9ee892544ec., storeName=f3ed23ac403772e0fe4ca9ee892544ec/A, priority=13, startTime=1731751158939; duration=0sec 2024-11-16T09:59:19,389 DEBUG [RS:0;037c58a19ddb:37267-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-16T09:59:19,389 DEBUG [RS:0;037c58a19ddb:37267-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: f3ed23ac403772e0fe4ca9ee892544ec:A 2024-11-16T09:59:19,393 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3ed23ac403772e0fe4ca9ee892544ec, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T09:59:19,394 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] ipc.CallRunner(138): callId: 130 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34408 deadline: 1731751219393, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3ed23ac403772e0fe4ca9ee892544ec, server=037c58a19ddb,37267,1731751077844 2024-11-16T09:59:19,394 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3ed23ac403772e0fe4ca9ee892544ec, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T09:59:19,394 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731751137553.f3ed23ac403772e0fe4ca9ee892544ec., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 , the old value is region=TestAcidGuarantees,,1731751137553.f3ed23ac403772e0fe4ca9ee892544ec., hostname=037c58a19ddb,37267,1731751077844, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3ed23ac403772e0fe4ca9ee892544ec, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:59:19,394 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] ipc.CallRunner(138): callId: 125 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34414 deadline: 1731751219393, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3ed23ac403772e0fe4ca9ee892544ec, server=037c58a19ddb,37267,1731751077844 2024-11-16T09:59:19,394 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731751137553.f3ed23ac403772e0fe4ca9ee892544ec., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3ed23ac403772e0fe4ca9ee892544ec, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:59:19,394 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731751137553.f3ed23ac403772e0fe4ca9ee892544ec., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 because the exception is null or not the one we care about 2024-11-16T09:59:19,394 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731751137553.f3ed23ac403772e0fe4ca9ee892544ec., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 , the old value is region=TestAcidGuarantees,,1731751137553.f3ed23ac403772e0fe4ca9ee892544ec., hostname=037c58a19ddb,37267,1731751077844, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3ed23ac403772e0fe4ca9ee892544ec, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:59:19,394 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731751137553.f3ed23ac403772e0fe4ca9ee892544ec., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3ed23ac403772e0fe4ca9ee892544ec, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:59:19,394 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731751137553.f3ed23ac403772e0fe4ca9ee892544ec., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 because the exception is null or not the one we care about 2024-11-16T09:59:19,409 INFO [RS_FLUSH_OPERATIONS-regionserver/037c58a19ddb:0-1 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=394 (bloomFilter=true), to=hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/f3ed23ac403772e0fe4ca9ee892544ec/.tmp/A/c80e5c0c86dc40869a7ee85491ab01c0 2024-11-16T09:59:19,416 DEBUG [RS_FLUSH_OPERATIONS-regionserver/037c58a19ddb:0-1 {event_type=RS_FLUSH_REGIONS, pid=83}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/f3ed23ac403772e0fe4ca9ee892544ec/.tmp/B/2da0bbf876be43f8828f0312f724aada is 50, key is test_row_0/B:col10/1731751158118/Put/seqid=0 2024-11-16T09:59:19,420 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37591 is added to blk_1073742160_1336 (size=12301) 2024-11-16T09:59:19,420 INFO [RS_FLUSH_OPERATIONS-regionserver/037c58a19ddb:0-1 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=394 (bloomFilter=true), to=hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/f3ed23ac403772e0fe4ca9ee892544ec/.tmp/B/2da0bbf876be43f8828f0312f724aada 2024-11-16T09:59:19,427 DEBUG [RS_FLUSH_OPERATIONS-regionserver/037c58a19ddb:0-1 {event_type=RS_FLUSH_REGIONS, pid=83}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/f3ed23ac403772e0fe4ca9ee892544ec/.tmp/C/c97ad2e6a0da44c2ba953469d1ef4f35 is 50, key is test_row_0/C:col10/1731751158118/Put/seqid=0 2024-11-16T09:59:19,436 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37591 is added to blk_1073742161_1337 (size=12301) 2024-11-16T09:59:19,605 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3ed23ac403772e0fe4ca9ee892544ec, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T09:59:19,605 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] ipc.CallRunner(138): callId: 131 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34408 deadline: 1731751219602, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3ed23ac403772e0fe4ca9ee892544ec, server=037c58a19ddb,37267,1731751077844 2024-11-16T09:59:19,605 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3ed23ac403772e0fe4ca9ee892544ec, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T09:59:19,605 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731751137553.f3ed23ac403772e0fe4ca9ee892544ec., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 , the old value is region=TestAcidGuarantees,,1731751137553.f3ed23ac403772e0fe4ca9ee892544ec., hostname=037c58a19ddb,37267,1731751077844, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3ed23ac403772e0fe4ca9ee892544ec, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:59:19,605 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731751137553.f3ed23ac403772e0fe4ca9ee892544ec., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3ed23ac403772e0fe4ca9ee892544ec, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:59:19,605 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] ipc.CallRunner(138): callId: 126 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34414 deadline: 1731751219602, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3ed23ac403772e0fe4ca9ee892544ec, server=037c58a19ddb,37267,1731751077844 2024-11-16T09:59:19,605 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731751137553.f3ed23ac403772e0fe4ca9ee892544ec., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 because the exception is null or not the one we care about 2024-11-16T09:59:19,606 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731751137553.f3ed23ac403772e0fe4ca9ee892544ec., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 , the old value is region=TestAcidGuarantees,,1731751137553.f3ed23ac403772e0fe4ca9ee892544ec., hostname=037c58a19ddb,37267,1731751077844, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3ed23ac403772e0fe4ca9ee892544ec, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:59:19,606 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731751137553.f3ed23ac403772e0fe4ca9ee892544ec., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3ed23ac403772e0fe4ca9ee892544ec, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:59:19,606 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731751137553.f3ed23ac403772e0fe4ca9ee892544ec., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 because the exception is null or not the one we care about 2024-11-16T09:59:19,748 INFO [Thread-1192 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1192. 2024-11-16T09:59:19,748 DEBUG [Thread-1192 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-16T09:59:19,748 INFO [Thread-1188 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1188. 2024-11-16T09:59:19,748 DEBUG [Thread-1192 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T09:59:19,748 DEBUG [Thread-1188 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-16T09:59:19,748 DEBUG [Thread-1188 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T09:59:19,749 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-16T09:59:19,749 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-16T09:59:19,749 DEBUG [Thread-1192 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T09:59:19,749 DEBUG [Thread-1188 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T09:59:19,750 INFO [Thread-1190 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1190. 2024-11-16T09:59:19,750 DEBUG [Thread-1190 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-16T09:59:19,750 DEBUG [Thread-1190 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T09:59:19,750 DEBUG [Thread-1190 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T09:59:19,750 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-16T09:59:19,751 INFO [Thread-1196 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1196. 2024-11-16T09:59:19,751 DEBUG [Thread-1196 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-16T09:59:19,751 DEBUG [Thread-1196 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T09:59:19,751 DEBUG [Thread-1196 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T09:59:19,751 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-16T09:59:19,752 INFO [Thread-1194 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1194. 2024-11-16T09:59:19,752 DEBUG [Thread-1194 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-16T09:59:19,752 DEBUG [Thread-1194 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T09:59:19,752 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-16T09:59:19,752 DEBUG [Thread-1194 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T09:59:19,834 INFO [RS_FLUSH_OPERATIONS-regionserver/037c58a19ddb:0-1 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=394 (bloomFilter=true), to=hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/f3ed23ac403772e0fe4ca9ee892544ec/.tmp/C/c97ad2e6a0da44c2ba953469d1ef4f35 2024-11-16T09:59:19,838 DEBUG [RS_FLUSH_OPERATIONS-regionserver/037c58a19ddb:0-1 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/f3ed23ac403772e0fe4ca9ee892544ec/.tmp/A/c80e5c0c86dc40869a7ee85491ab01c0 as hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/f3ed23ac403772e0fe4ca9ee892544ec/A/c80e5c0c86dc40869a7ee85491ab01c0 2024-11-16T09:59:19,841 INFO [RS_FLUSH_OPERATIONS-regionserver/037c58a19ddb:0-1 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/f3ed23ac403772e0fe4ca9ee892544ec/A/c80e5c0c86dc40869a7ee85491ab01c0, entries=150, sequenceid=394, filesize=12.0 K 2024-11-16T09:59:19,842 DEBUG [RS_FLUSH_OPERATIONS-regionserver/037c58a19ddb:0-1 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/f3ed23ac403772e0fe4ca9ee892544ec/.tmp/B/2da0bbf876be43f8828f0312f724aada as hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/f3ed23ac403772e0fe4ca9ee892544ec/B/2da0bbf876be43f8828f0312f724aada 2024-11-16T09:59:19,845 INFO [RS_FLUSH_OPERATIONS-regionserver/037c58a19ddb:0-1 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/f3ed23ac403772e0fe4ca9ee892544ec/B/2da0bbf876be43f8828f0312f724aada, entries=150, sequenceid=394, filesize=12.0 K 2024-11-16T09:59:19,846 DEBUG [RS_FLUSH_OPERATIONS-regionserver/037c58a19ddb:0-1 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/f3ed23ac403772e0fe4ca9ee892544ec/.tmp/C/c97ad2e6a0da44c2ba953469d1ef4f35 as hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/f3ed23ac403772e0fe4ca9ee892544ec/C/c97ad2e6a0da44c2ba953469d1ef4f35 2024-11-16T09:59:19,849 INFO [RS_FLUSH_OPERATIONS-regionserver/037c58a19ddb:0-1 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/f3ed23ac403772e0fe4ca9ee892544ec/C/c97ad2e6a0da44c2ba953469d1ef4f35, entries=150, sequenceid=394, filesize=12.0 K 2024-11-16T09:59:19,850 INFO [RS_FLUSH_OPERATIONS-regionserver/037c58a19ddb:0-1 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.HRegion(3140): Finished flush of dataSize ~147.60 KB/151140, heapSize ~387.42 KB/396720, currentSize=60.38 KB/61830 for f3ed23ac403772e0fe4ca9ee892544ec in 858ms, sequenceid=394, compaction requested=true 2024-11-16T09:59:19,850 DEBUG [RS_FLUSH_OPERATIONS-regionserver/037c58a19ddb:0-1 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.HRegion(2603): Flush status journal for f3ed23ac403772e0fe4ca9ee892544ec: 2024-11-16T09:59:19,850 DEBUG [RS_FLUSH_OPERATIONS-regionserver/037c58a19ddb:0-1 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731751137553.f3ed23ac403772e0fe4ca9ee892544ec. 2024-11-16T09:59:19,850 DEBUG [RS_FLUSH_OPERATIONS-regionserver/037c58a19ddb:0-1 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=83 2024-11-16T09:59:19,850 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37877 {}] master.HMaster(4169): Remote procedure done, pid=83 2024-11-16T09:59:19,853 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=83, resume processing ppid=82 2024-11-16T09:59:19,853 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=83, ppid=82, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.6240 sec 2024-11-16T09:59:19,855 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=82, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=82, table=TestAcidGuarantees in 1.6300 sec 2024-11-16T09:59:19,914 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] regionserver.HRegion(8855): Flush requested on f3ed23ac403772e0fe4ca9ee892544ec 2024-11-16T09:59:19,914 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-16T09:59:19,914 INFO [Thread-1181 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1181. 2024-11-16T09:59:19,914 INFO [Thread-1185 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1185. 2024-11-16T09:59:19,914 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing f3ed23ac403772e0fe4ca9ee892544ec 3/3 column families, dataSize=73.80 KB heapSize=194.11 KB 2024-11-16T09:59:19,914 DEBUG [Thread-1181 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-16T09:59:19,914 DEBUG [Thread-1185 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-16T09:59:19,914 DEBUG [Thread-1181 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T09:59:19,914 DEBUG [Thread-1185 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T09:59:19,915 DEBUG [Thread-1185 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T09:59:19,915 DEBUG [Thread-1181 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T09:59:19,915 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-16T09:59:19,915 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-16T09:59:19,918 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/f3ed23ac403772e0fe4ca9ee892544ec/.tmp/A/e1ba74d1fb694d7ca6d71ad6778d6ac6 is 50, key is test_row_0/A:col10/1731751159286/Put/seqid=0 2024-11-16T09:59:19,921 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37591 is added to blk_1073742162_1338 (size=12301) 2024-11-16T09:59:20,322 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=409 (bloomFilter=true), to=hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/f3ed23ac403772e0fe4ca9ee892544ec/.tmp/A/e1ba74d1fb694d7ca6d71ad6778d6ac6 2024-11-16T09:59:20,328 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/f3ed23ac403772e0fe4ca9ee892544ec/.tmp/B/a68d4c6cd7d542f7b49c6bc68a97ed7e is 50, key is test_row_0/B:col10/1731751159286/Put/seqid=0 2024-11-16T09:59:20,331 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37591 is added to blk_1073742163_1339 (size=12301) 2024-11-16T09:59:20,354 INFO [Thread-1183 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1183. 2024-11-16T09:59:20,354 DEBUG [Thread-1183 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-16T09:59:20,354 DEBUG [Thread-1183 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T09:59:20,354 DEBUG [Thread-1183 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T09:59:20,354 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-16T09:59:20,364 INFO [Thread-1177 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1177. 2024-11-16T09:59:20,364 INFO [Thread-1179 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1179. 2024-11-16T09:59:20,364 DEBUG [Thread-1177 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-16T09:59:20,364 DEBUG [Thread-1179 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-16T09:59:20,364 DEBUG [Thread-1177 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T09:59:20,364 DEBUG [Thread-1179 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T09:59:20,364 DEBUG [Thread-1177 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T09:59:20,364 DEBUG [Thread-1179 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T09:59:20,364 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-16T09:59:20,364 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-16T09:59:20,373 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37877 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=82 2024-11-16T09:59:20,373 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-16T09:59:20,373 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(392): Finished test. Writers: 2024-11-16T09:59:20,373 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 55 2024-11-16T09:59:20,373 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 66 2024-11-16T09:59:20,373 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 73 2024-11-16T09:59:20,373 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 61 2024-11-16T09:59:20,373 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 68 2024-11-16T09:59:20,373 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(396): Readers: 2024-11-16T09:59:20,373 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 4571 2024-11-16T09:59:20,373 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 4509 2024-11-16T09:59:20,373 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 4207 2024-11-16T09:59:20,373 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 4589 2024-11-16T09:59:20,373 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 4504 2024-11-16T09:59:20,373 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(400): Scanners: 2024-11-16T09:59:20,374 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-16T09:59:20,374 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool.doWork(AcidGuaranteesTestTool.java:131) at org.apache.hadoop.hbase.util.AbstractHBaseTool.run(AbstractHBaseTool.java:150) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.runTestAtomicity(AcidGuaranteesTestBase.java:102) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.runTestAtomicity(AcidGuaranteesTestBase.java:91) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.testGetAtomicity(AcidGuaranteesTestBase.java:107) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-16T09:59:20,374 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T09:59:20,374 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T09:59:20,374 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-16T09:59:20,374 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37877 {}] master.HMaster$13(2820): Client=jenkins//172.17.0.2 disable TestAcidGuarantees 2024-11-16T09:59:20,375 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37877 {}] procedure2.ProcedureExecutor(1139): Stored pid=84, state=RUNNABLE:DISABLE_TABLE_PREPARE, hasLock=false; DisableTableProcedure table=TestAcidGuarantees 2024-11-16T09:59:20,377 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37877 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=84 2024-11-16T09:59:20,377 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731751160377"}]},"ts":"1731751160377"} 2024-11-16T09:59:20,378 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLING in hbase:meta 2024-11-16T09:59:20,378 INFO [PEWorker-5 {}] procedure.DisableTableProcedure(284): Set TestAcidGuarantees to state=DISABLING 2024-11-16T09:59:20,379 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=85, ppid=84, state=RUNNABLE:CLOSE_TABLE_REGIONS_SCHEDULE, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees}] 2024-11-16T09:59:20,380 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=86, ppid=85, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=f3ed23ac403772e0fe4ca9ee892544ec, UNASSIGN}] 2024-11-16T09:59:20,381 INFO [PEWorker-3 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=86, ppid=85, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=f3ed23ac403772e0fe4ca9ee892544ec, UNASSIGN 2024-11-16T09:59:20,381 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=86 updating hbase:meta row=f3ed23ac403772e0fe4ca9ee892544ec, regionState=CLOSING, regionLocation=037c58a19ddb,37267,1731751077844 2024-11-16T09:59:20,382 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=86, ppid=85, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=f3ed23ac403772e0fe4ca9ee892544ec, UNASSIGN because future has completed 2024-11-16T09:59:20,383 DEBUG [PEWorker-2 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-11-16T09:59:20,383 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=87, ppid=86, state=RUNNABLE, hasLock=false; CloseRegionProcedure f3ed23ac403772e0fe4ca9ee892544ec, server=037c58a19ddb,37267,1731751077844}] 2024-11-16T09:59:20,483 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37877 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=84 2024-11-16T09:59:20,535 INFO [RS_CLOSE_REGION-regionserver/037c58a19ddb:0-0 {event_type=M_RS_CLOSE_REGION, pid=87}] handler.UnassignRegionHandler(122): Close f3ed23ac403772e0fe4ca9ee892544ec 2024-11-16T09:59:20,535 DEBUG [RS_CLOSE_REGION-regionserver/037c58a19ddb:0-0 {event_type=M_RS_CLOSE_REGION, pid=87}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-11-16T09:59:20,535 DEBUG [RS_CLOSE_REGION-regionserver/037c58a19ddb:0-0 {event_type=M_RS_CLOSE_REGION, pid=87}] regionserver.HRegion(1722): Closing f3ed23ac403772e0fe4ca9ee892544ec, disabling compactions & flushes 2024-11-16T09:59:20,535 DEBUG [RS_CLOSE_REGION-regionserver/037c58a19ddb:0-0 {event_type=M_RS_CLOSE_REGION, pid=87}] regionserver.HRegion(1993): waiting for 0 compactions & cache flush to complete for region TestAcidGuarantees,,1731751137553.f3ed23ac403772e0fe4ca9ee892544ec. 2024-11-16T09:59:20,693 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37877 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=84 2024-11-16T09:59:20,732 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=409 (bloomFilter=true), to=hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/f3ed23ac403772e0fe4ca9ee892544ec/.tmp/B/a68d4c6cd7d542f7b49c6bc68a97ed7e 2024-11-16T09:59:20,738 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/f3ed23ac403772e0fe4ca9ee892544ec/.tmp/C/5974b10c166b4ed6ac601358bb28f686 is 50, key is test_row_0/C:col10/1731751159286/Put/seqid=0 2024-11-16T09:59:20,741 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37591 is added to blk_1073742164_1340 (size=12301) 2024-11-16T09:59:21,003 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37877 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=84 2024-11-16T09:59:21,142 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=409 (bloomFilter=true), to=hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/f3ed23ac403772e0fe4ca9ee892544ec/.tmp/C/5974b10c166b4ed6ac601358bb28f686 2024-11-16T09:59:21,146 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/f3ed23ac403772e0fe4ca9ee892544ec/.tmp/A/e1ba74d1fb694d7ca6d71ad6778d6ac6 as hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/f3ed23ac403772e0fe4ca9ee892544ec/A/e1ba74d1fb694d7ca6d71ad6778d6ac6 2024-11-16T09:59:21,149 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/f3ed23ac403772e0fe4ca9ee892544ec/A/e1ba74d1fb694d7ca6d71ad6778d6ac6, entries=150, sequenceid=409, filesize=12.0 K 2024-11-16T09:59:21,150 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/f3ed23ac403772e0fe4ca9ee892544ec/.tmp/B/a68d4c6cd7d542f7b49c6bc68a97ed7e as hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/f3ed23ac403772e0fe4ca9ee892544ec/B/a68d4c6cd7d542f7b49c6bc68a97ed7e 2024-11-16T09:59:21,153 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/f3ed23ac403772e0fe4ca9ee892544ec/B/a68d4c6cd7d542f7b49c6bc68a97ed7e, entries=150, sequenceid=409, filesize=12.0 K 2024-11-16T09:59:21,153 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/f3ed23ac403772e0fe4ca9ee892544ec/.tmp/C/5974b10c166b4ed6ac601358bb28f686 as hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/f3ed23ac403772e0fe4ca9ee892544ec/C/5974b10c166b4ed6ac601358bb28f686 2024-11-16T09:59:21,156 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/f3ed23ac403772e0fe4ca9ee892544ec/C/5974b10c166b4ed6ac601358bb28f686, entries=150, sequenceid=409, filesize=12.0 K 2024-11-16T09:59:21,157 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~73.80 KB/75570, heapSize ~194.06 KB/198720, currentSize=20.13 KB/20610 for f3ed23ac403772e0fe4ca9ee892544ec in 1243ms, sequenceid=409, compaction requested=true 2024-11-16T09:59:21,157 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for f3ed23ac403772e0fe4ca9ee892544ec: 2024-11-16T09:59:21,157 INFO [RS_CLOSE_REGION-regionserver/037c58a19ddb:0-0 {event_type=M_RS_CLOSE_REGION, pid=87}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1731751137553.f3ed23ac403772e0fe4ca9ee892544ec. 2024-11-16T09:59:21,157 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store f3ed23ac403772e0fe4ca9ee892544ec:A, priority=-2147483648, current under compaction store size is 1 2024-11-16T09:59:21,157 DEBUG [RS_CLOSE_REGION-regionserver/037c58a19ddb:0-0 {event_type=M_RS_CLOSE_REGION, pid=87}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1731751137553.f3ed23ac403772e0fe4ca9ee892544ec. 2024-11-16T09:59:21,157 DEBUG [RS:0;037c58a19ddb:37267-shortCompactions-0 {}] regionserver.CompactSplit(450): Not compacting TestAcidGuarantees,,1731751137553.f3ed23ac403772e0fe4ca9ee892544ec. because compaction request was cancelled 2024-11-16T09:59:21,157 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-16T09:59:21,157 DEBUG [RS_CLOSE_REGION-regionserver/037c58a19ddb:0-0 {event_type=M_RS_CLOSE_REGION, pid=87}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1731751137553.f3ed23ac403772e0fe4ca9ee892544ec. after waiting 0 ms 2024-11-16T09:59:21,157 DEBUG [RS:0;037c58a19ddb:37267-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: f3ed23ac403772e0fe4ca9ee892544ec:A 2024-11-16T09:59:21,157 DEBUG [RS_CLOSE_REGION-regionserver/037c58a19ddb:0-0 {event_type=M_RS_CLOSE_REGION, pid=87}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1731751137553.f3ed23ac403772e0fe4ca9ee892544ec. 2024-11-16T09:59:21,157 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store f3ed23ac403772e0fe4ca9ee892544ec:B, priority=-2147483648, current under compaction store size is 1 2024-11-16T09:59:21,157 DEBUG [RS:0;037c58a19ddb:37267-shortCompactions-0 {}] regionserver.CompactSplit(450): Not compacting TestAcidGuarantees,,1731751137553.f3ed23ac403772e0fe4ca9ee892544ec. because compaction request was cancelled 2024-11-16T09:59:21,157 DEBUG [RS:0;037c58a19ddb:37267-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: f3ed23ac403772e0fe4ca9ee892544ec:B 2024-11-16T09:59:21,157 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-16T09:59:21,157 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store f3ed23ac403772e0fe4ca9ee892544ec:C, priority=-2147483648, current under compaction store size is 1 2024-11-16T09:59:21,157 DEBUG [RS:0;037c58a19ddb:37267-shortCompactions-0 {}] regionserver.CompactSplit(450): Not compacting TestAcidGuarantees,,1731751137553.f3ed23ac403772e0fe4ca9ee892544ec. because compaction request was cancelled 2024-11-16T09:59:21,157 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-16T09:59:21,157 DEBUG [RS:0;037c58a19ddb:37267-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: f3ed23ac403772e0fe4ca9ee892544ec:C 2024-11-16T09:59:21,157 INFO [RS_CLOSE_REGION-regionserver/037c58a19ddb:0-0 {event_type=M_RS_CLOSE_REGION, pid=87}] regionserver.HRegion(2902): Flushing f3ed23ac403772e0fe4ca9ee892544ec 3/3 column families, dataSize=20.13 KB heapSize=53.48 KB 2024-11-16T09:59:21,161 DEBUG [RS_CLOSE_REGION-regionserver/037c58a19ddb:0-0 {event_type=M_RS_CLOSE_REGION, pid=87}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/f3ed23ac403772e0fe4ca9ee892544ec/.tmp/A/f23a43b0b66f42ef8081f7a02159bd86 is 50, key is test_row_0/A:col10/1731751160363/Put/seqid=0 2024-11-16T09:59:21,164 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37591 is added to blk_1073742165_1341 (size=9857) 2024-11-16T09:59:21,513 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37877 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=84 2024-11-16T09:59:21,565 INFO [RS_CLOSE_REGION-regionserver/037c58a19ddb:0-0 {event_type=M_RS_CLOSE_REGION, pid=87}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=6.71 KB at sequenceid=415 (bloomFilter=true), to=hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/f3ed23ac403772e0fe4ca9ee892544ec/.tmp/A/f23a43b0b66f42ef8081f7a02159bd86 2024-11-16T09:59:21,571 DEBUG [RS_CLOSE_REGION-regionserver/037c58a19ddb:0-0 {event_type=M_RS_CLOSE_REGION, pid=87}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/f3ed23ac403772e0fe4ca9ee892544ec/.tmp/B/b56e0226632242a8b999c8e3e32b7f63 is 50, key is test_row_0/B:col10/1731751160363/Put/seqid=0 2024-11-16T09:59:21,574 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37591 is added to blk_1073742166_1342 (size=9857) 2024-11-16T09:59:21,975 INFO [RS_CLOSE_REGION-regionserver/037c58a19ddb:0-0 {event_type=M_RS_CLOSE_REGION, pid=87}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=6.71 KB at sequenceid=415 (bloomFilter=true), to=hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/f3ed23ac403772e0fe4ca9ee892544ec/.tmp/B/b56e0226632242a8b999c8e3e32b7f63 2024-11-16T09:59:21,981 DEBUG [RS_CLOSE_REGION-regionserver/037c58a19ddb:0-0 {event_type=M_RS_CLOSE_REGION, pid=87}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/f3ed23ac403772e0fe4ca9ee892544ec/.tmp/C/03d27bbfa32d4a2d8ec7125bef3bb47c is 50, key is test_row_0/C:col10/1731751160363/Put/seqid=0 2024-11-16T09:59:21,984 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37591 is added to blk_1073742167_1343 (size=9857) 2024-11-16T09:59:22,385 INFO [RS_CLOSE_REGION-regionserver/037c58a19ddb:0-0 {event_type=M_RS_CLOSE_REGION, pid=87}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=6.71 KB at sequenceid=415 (bloomFilter=true), to=hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/f3ed23ac403772e0fe4ca9ee892544ec/.tmp/C/03d27bbfa32d4a2d8ec7125bef3bb47c 2024-11-16T09:59:22,390 DEBUG [RS_CLOSE_REGION-regionserver/037c58a19ddb:0-0 {event_type=M_RS_CLOSE_REGION, pid=87}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/f3ed23ac403772e0fe4ca9ee892544ec/.tmp/A/f23a43b0b66f42ef8081f7a02159bd86 as hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/f3ed23ac403772e0fe4ca9ee892544ec/A/f23a43b0b66f42ef8081f7a02159bd86 2024-11-16T09:59:22,393 INFO [RS_CLOSE_REGION-regionserver/037c58a19ddb:0-0 {event_type=M_RS_CLOSE_REGION, pid=87}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/f3ed23ac403772e0fe4ca9ee892544ec/A/f23a43b0b66f42ef8081f7a02159bd86, entries=100, sequenceid=415, filesize=9.6 K 2024-11-16T09:59:22,394 DEBUG [RS_CLOSE_REGION-regionserver/037c58a19ddb:0-0 {event_type=M_RS_CLOSE_REGION, pid=87}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/f3ed23ac403772e0fe4ca9ee892544ec/.tmp/B/b56e0226632242a8b999c8e3e32b7f63 as hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/f3ed23ac403772e0fe4ca9ee892544ec/B/b56e0226632242a8b999c8e3e32b7f63 2024-11-16T09:59:22,397 INFO [RS_CLOSE_REGION-regionserver/037c58a19ddb:0-0 {event_type=M_RS_CLOSE_REGION, pid=87}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/f3ed23ac403772e0fe4ca9ee892544ec/B/b56e0226632242a8b999c8e3e32b7f63, entries=100, sequenceid=415, filesize=9.6 K 2024-11-16T09:59:22,398 DEBUG [RS_CLOSE_REGION-regionserver/037c58a19ddb:0-0 {event_type=M_RS_CLOSE_REGION, pid=87}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/f3ed23ac403772e0fe4ca9ee892544ec/.tmp/C/03d27bbfa32d4a2d8ec7125bef3bb47c as hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/f3ed23ac403772e0fe4ca9ee892544ec/C/03d27bbfa32d4a2d8ec7125bef3bb47c 2024-11-16T09:59:22,401 INFO [RS_CLOSE_REGION-regionserver/037c58a19ddb:0-0 {event_type=M_RS_CLOSE_REGION, pid=87}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/f3ed23ac403772e0fe4ca9ee892544ec/C/03d27bbfa32d4a2d8ec7125bef3bb47c, entries=100, sequenceid=415, filesize=9.6 K 2024-11-16T09:59:22,401 INFO [RS_CLOSE_REGION-regionserver/037c58a19ddb:0-0 {event_type=M_RS_CLOSE_REGION, pid=87}] regionserver.HRegion(3140): Finished flush of dataSize ~20.13 KB/20610, heapSize ~53.44 KB/54720, currentSize=0 B/0 for f3ed23ac403772e0fe4ca9ee892544ec in 1244ms, sequenceid=415, compaction requested=true 2024-11-16T09:59:22,402 DEBUG [StoreCloser-TestAcidGuarantees,,1731751137553.f3ed23ac403772e0fe4ca9ee892544ec.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/f3ed23ac403772e0fe4ca9ee892544ec/A/c97cbf9e65af4cc4afa063288dd53d11, hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/f3ed23ac403772e0fe4ca9ee892544ec/A/c3e74913af05458d9452010d8e8044c0, hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/f3ed23ac403772e0fe4ca9ee892544ec/A/3dc937ecac70408cba3ef81b89802dd1, hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/f3ed23ac403772e0fe4ca9ee892544ec/A/a4ceccf438df4ea487b2d78dc8baaad9, hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/f3ed23ac403772e0fe4ca9ee892544ec/A/c4ef90a8928e4ab49e0b5f591292b9b9, hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/f3ed23ac403772e0fe4ca9ee892544ec/A/699d58ef21354836b7b3485f7bcfb2c0, hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/f3ed23ac403772e0fe4ca9ee892544ec/A/fc6aac9ad19e442cbbab9d08b96d6884, hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/f3ed23ac403772e0fe4ca9ee892544ec/A/27f0ac8785f94a86aedae7fb6146d397, hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/f3ed23ac403772e0fe4ca9ee892544ec/A/ae5eee1b468f4af7a61d7367b1991b0b, hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/f3ed23ac403772e0fe4ca9ee892544ec/A/f9b5d8015d1b4e76a61f9ad1279908b4, hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/f3ed23ac403772e0fe4ca9ee892544ec/A/d241f3b4a9f54d6eaf9c03b0d64cdbb6, hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/f3ed23ac403772e0fe4ca9ee892544ec/A/726ccf100eb646049bd2e286da649ab2, hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/f3ed23ac403772e0fe4ca9ee892544ec/A/2a41a1f803ea4e6e9929313e235b0236, hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/f3ed23ac403772e0fe4ca9ee892544ec/A/e1f68d6740d84ab2b22314b6666d769e, hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/f3ed23ac403772e0fe4ca9ee892544ec/A/1192a667db6a468d8f92ad86a890bac9, hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/f3ed23ac403772e0fe4ca9ee892544ec/A/60896e91c5674529aaf3a78f98e92f39, hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/f3ed23ac403772e0fe4ca9ee892544ec/A/1775c312f3b740ccb57738318487b1ee, hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/f3ed23ac403772e0fe4ca9ee892544ec/A/8102fd38ad8549e2a0a14cc8eef8c537, hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/f3ed23ac403772e0fe4ca9ee892544ec/A/882d0e38ac3f4f40bbb1b64db18c4a2d, hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/f3ed23ac403772e0fe4ca9ee892544ec/A/12aace3b248347fc94ca8f8143923c21, hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/f3ed23ac403772e0fe4ca9ee892544ec/A/c2ff2a2750b14c829383d875cf110f93, hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/f3ed23ac403772e0fe4ca9ee892544ec/A/b8255e158316461c9e6faf3a1fcf57f1, hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/f3ed23ac403772e0fe4ca9ee892544ec/A/7edbb0569e5c4e04bfa1b45acf180081, hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/f3ed23ac403772e0fe4ca9ee892544ec/A/b097f1e4754c45e7863be4f0cb423c62, hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/f3ed23ac403772e0fe4ca9ee892544ec/A/d07186c35def4c13966a8948333333ac, hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/f3ed23ac403772e0fe4ca9ee892544ec/A/1e115d5e923f4de2a49c0fc719d77e9c] to archive 2024-11-16T09:59:22,403 DEBUG [StoreCloser-TestAcidGuarantees,,1731751137553.f3ed23ac403772e0fe4ca9ee892544ec.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-16T09:59:22,404 DEBUG [StoreCloser-TestAcidGuarantees,,1731751137553.f3ed23ac403772e0fe4ca9ee892544ec.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/f3ed23ac403772e0fe4ca9ee892544ec/A/c97cbf9e65af4cc4afa063288dd53d11 to hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/archive/data/default/TestAcidGuarantees/f3ed23ac403772e0fe4ca9ee892544ec/A/c97cbf9e65af4cc4afa063288dd53d11 2024-11-16T09:59:22,405 DEBUG [StoreCloser-TestAcidGuarantees,,1731751137553.f3ed23ac403772e0fe4ca9ee892544ec.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/f3ed23ac403772e0fe4ca9ee892544ec/A/c3e74913af05458d9452010d8e8044c0 to hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/archive/data/default/TestAcidGuarantees/f3ed23ac403772e0fe4ca9ee892544ec/A/c3e74913af05458d9452010d8e8044c0 2024-11-16T09:59:22,406 DEBUG [StoreCloser-TestAcidGuarantees,,1731751137553.f3ed23ac403772e0fe4ca9ee892544ec.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/f3ed23ac403772e0fe4ca9ee892544ec/A/3dc937ecac70408cba3ef81b89802dd1 to hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/archive/data/default/TestAcidGuarantees/f3ed23ac403772e0fe4ca9ee892544ec/A/3dc937ecac70408cba3ef81b89802dd1 2024-11-16T09:59:22,407 DEBUG [StoreCloser-TestAcidGuarantees,,1731751137553.f3ed23ac403772e0fe4ca9ee892544ec.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/f3ed23ac403772e0fe4ca9ee892544ec/A/a4ceccf438df4ea487b2d78dc8baaad9 to hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/archive/data/default/TestAcidGuarantees/f3ed23ac403772e0fe4ca9ee892544ec/A/a4ceccf438df4ea487b2d78dc8baaad9 2024-11-16T09:59:22,408 DEBUG [StoreCloser-TestAcidGuarantees,,1731751137553.f3ed23ac403772e0fe4ca9ee892544ec.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/f3ed23ac403772e0fe4ca9ee892544ec/A/c4ef90a8928e4ab49e0b5f591292b9b9 to hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/archive/data/default/TestAcidGuarantees/f3ed23ac403772e0fe4ca9ee892544ec/A/c4ef90a8928e4ab49e0b5f591292b9b9 2024-11-16T09:59:22,409 DEBUG [StoreCloser-TestAcidGuarantees,,1731751137553.f3ed23ac403772e0fe4ca9ee892544ec.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/f3ed23ac403772e0fe4ca9ee892544ec/A/699d58ef21354836b7b3485f7bcfb2c0 to hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/archive/data/default/TestAcidGuarantees/f3ed23ac403772e0fe4ca9ee892544ec/A/699d58ef21354836b7b3485f7bcfb2c0 2024-11-16T09:59:22,410 DEBUG [StoreCloser-TestAcidGuarantees,,1731751137553.f3ed23ac403772e0fe4ca9ee892544ec.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/f3ed23ac403772e0fe4ca9ee892544ec/A/fc6aac9ad19e442cbbab9d08b96d6884 to hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/archive/data/default/TestAcidGuarantees/f3ed23ac403772e0fe4ca9ee892544ec/A/fc6aac9ad19e442cbbab9d08b96d6884 2024-11-16T09:59:22,410 DEBUG [StoreCloser-TestAcidGuarantees,,1731751137553.f3ed23ac403772e0fe4ca9ee892544ec.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/f3ed23ac403772e0fe4ca9ee892544ec/A/27f0ac8785f94a86aedae7fb6146d397 to hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/archive/data/default/TestAcidGuarantees/f3ed23ac403772e0fe4ca9ee892544ec/A/27f0ac8785f94a86aedae7fb6146d397 2024-11-16T09:59:22,411 DEBUG [StoreCloser-TestAcidGuarantees,,1731751137553.f3ed23ac403772e0fe4ca9ee892544ec.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/f3ed23ac403772e0fe4ca9ee892544ec/A/ae5eee1b468f4af7a61d7367b1991b0b to hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/archive/data/default/TestAcidGuarantees/f3ed23ac403772e0fe4ca9ee892544ec/A/ae5eee1b468f4af7a61d7367b1991b0b 2024-11-16T09:59:22,412 DEBUG [StoreCloser-TestAcidGuarantees,,1731751137553.f3ed23ac403772e0fe4ca9ee892544ec.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/f3ed23ac403772e0fe4ca9ee892544ec/A/f9b5d8015d1b4e76a61f9ad1279908b4 to hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/archive/data/default/TestAcidGuarantees/f3ed23ac403772e0fe4ca9ee892544ec/A/f9b5d8015d1b4e76a61f9ad1279908b4 2024-11-16T09:59:22,413 DEBUG [StoreCloser-TestAcidGuarantees,,1731751137553.f3ed23ac403772e0fe4ca9ee892544ec.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/f3ed23ac403772e0fe4ca9ee892544ec/A/d241f3b4a9f54d6eaf9c03b0d64cdbb6 to hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/archive/data/default/TestAcidGuarantees/f3ed23ac403772e0fe4ca9ee892544ec/A/d241f3b4a9f54d6eaf9c03b0d64cdbb6 2024-11-16T09:59:22,414 DEBUG [StoreCloser-TestAcidGuarantees,,1731751137553.f3ed23ac403772e0fe4ca9ee892544ec.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/f3ed23ac403772e0fe4ca9ee892544ec/A/726ccf100eb646049bd2e286da649ab2 to hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/archive/data/default/TestAcidGuarantees/f3ed23ac403772e0fe4ca9ee892544ec/A/726ccf100eb646049bd2e286da649ab2 2024-11-16T09:59:22,415 DEBUG [StoreCloser-TestAcidGuarantees,,1731751137553.f3ed23ac403772e0fe4ca9ee892544ec.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/f3ed23ac403772e0fe4ca9ee892544ec/A/2a41a1f803ea4e6e9929313e235b0236 to hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/archive/data/default/TestAcidGuarantees/f3ed23ac403772e0fe4ca9ee892544ec/A/2a41a1f803ea4e6e9929313e235b0236 2024-11-16T09:59:22,415 DEBUG [StoreCloser-TestAcidGuarantees,,1731751137553.f3ed23ac403772e0fe4ca9ee892544ec.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/f3ed23ac403772e0fe4ca9ee892544ec/A/e1f68d6740d84ab2b22314b6666d769e to hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/archive/data/default/TestAcidGuarantees/f3ed23ac403772e0fe4ca9ee892544ec/A/e1f68d6740d84ab2b22314b6666d769e 2024-11-16T09:59:22,416 DEBUG [StoreCloser-TestAcidGuarantees,,1731751137553.f3ed23ac403772e0fe4ca9ee892544ec.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/f3ed23ac403772e0fe4ca9ee892544ec/A/1192a667db6a468d8f92ad86a890bac9 to hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/archive/data/default/TestAcidGuarantees/f3ed23ac403772e0fe4ca9ee892544ec/A/1192a667db6a468d8f92ad86a890bac9 2024-11-16T09:59:22,417 DEBUG [StoreCloser-TestAcidGuarantees,,1731751137553.f3ed23ac403772e0fe4ca9ee892544ec.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/f3ed23ac403772e0fe4ca9ee892544ec/A/60896e91c5674529aaf3a78f98e92f39 to hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/archive/data/default/TestAcidGuarantees/f3ed23ac403772e0fe4ca9ee892544ec/A/60896e91c5674529aaf3a78f98e92f39 2024-11-16T09:59:22,418 DEBUG [StoreCloser-TestAcidGuarantees,,1731751137553.f3ed23ac403772e0fe4ca9ee892544ec.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/f3ed23ac403772e0fe4ca9ee892544ec/A/1775c312f3b740ccb57738318487b1ee to hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/archive/data/default/TestAcidGuarantees/f3ed23ac403772e0fe4ca9ee892544ec/A/1775c312f3b740ccb57738318487b1ee 2024-11-16T09:59:22,419 DEBUG [StoreCloser-TestAcidGuarantees,,1731751137553.f3ed23ac403772e0fe4ca9ee892544ec.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/f3ed23ac403772e0fe4ca9ee892544ec/A/8102fd38ad8549e2a0a14cc8eef8c537 to hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/archive/data/default/TestAcidGuarantees/f3ed23ac403772e0fe4ca9ee892544ec/A/8102fd38ad8549e2a0a14cc8eef8c537 2024-11-16T09:59:22,420 DEBUG [StoreCloser-TestAcidGuarantees,,1731751137553.f3ed23ac403772e0fe4ca9ee892544ec.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/f3ed23ac403772e0fe4ca9ee892544ec/A/882d0e38ac3f4f40bbb1b64db18c4a2d to hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/archive/data/default/TestAcidGuarantees/f3ed23ac403772e0fe4ca9ee892544ec/A/882d0e38ac3f4f40bbb1b64db18c4a2d 2024-11-16T09:59:22,421 DEBUG [StoreCloser-TestAcidGuarantees,,1731751137553.f3ed23ac403772e0fe4ca9ee892544ec.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/f3ed23ac403772e0fe4ca9ee892544ec/A/12aace3b248347fc94ca8f8143923c21 to hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/archive/data/default/TestAcidGuarantees/f3ed23ac403772e0fe4ca9ee892544ec/A/12aace3b248347fc94ca8f8143923c21 2024-11-16T09:59:22,422 DEBUG [StoreCloser-TestAcidGuarantees,,1731751137553.f3ed23ac403772e0fe4ca9ee892544ec.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/f3ed23ac403772e0fe4ca9ee892544ec/A/c2ff2a2750b14c829383d875cf110f93 to hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/archive/data/default/TestAcidGuarantees/f3ed23ac403772e0fe4ca9ee892544ec/A/c2ff2a2750b14c829383d875cf110f93 2024-11-16T09:59:22,422 DEBUG [StoreCloser-TestAcidGuarantees,,1731751137553.f3ed23ac403772e0fe4ca9ee892544ec.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/f3ed23ac403772e0fe4ca9ee892544ec/A/b8255e158316461c9e6faf3a1fcf57f1 to hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/archive/data/default/TestAcidGuarantees/f3ed23ac403772e0fe4ca9ee892544ec/A/b8255e158316461c9e6faf3a1fcf57f1 2024-11-16T09:59:22,423 DEBUG [StoreCloser-TestAcidGuarantees,,1731751137553.f3ed23ac403772e0fe4ca9ee892544ec.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/f3ed23ac403772e0fe4ca9ee892544ec/A/7edbb0569e5c4e04bfa1b45acf180081 to hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/archive/data/default/TestAcidGuarantees/f3ed23ac403772e0fe4ca9ee892544ec/A/7edbb0569e5c4e04bfa1b45acf180081 2024-11-16T09:59:22,424 DEBUG [StoreCloser-TestAcidGuarantees,,1731751137553.f3ed23ac403772e0fe4ca9ee892544ec.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/f3ed23ac403772e0fe4ca9ee892544ec/A/b097f1e4754c45e7863be4f0cb423c62 to hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/archive/data/default/TestAcidGuarantees/f3ed23ac403772e0fe4ca9ee892544ec/A/b097f1e4754c45e7863be4f0cb423c62 2024-11-16T09:59:22,425 DEBUG [StoreCloser-TestAcidGuarantees,,1731751137553.f3ed23ac403772e0fe4ca9ee892544ec.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/f3ed23ac403772e0fe4ca9ee892544ec/A/d07186c35def4c13966a8948333333ac to hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/archive/data/default/TestAcidGuarantees/f3ed23ac403772e0fe4ca9ee892544ec/A/d07186c35def4c13966a8948333333ac 2024-11-16T09:59:22,426 DEBUG [StoreCloser-TestAcidGuarantees,,1731751137553.f3ed23ac403772e0fe4ca9ee892544ec.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/f3ed23ac403772e0fe4ca9ee892544ec/A/1e115d5e923f4de2a49c0fc719d77e9c to hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/archive/data/default/TestAcidGuarantees/f3ed23ac403772e0fe4ca9ee892544ec/A/1e115d5e923f4de2a49c0fc719d77e9c 2024-11-16T09:59:22,428 DEBUG [StoreCloser-TestAcidGuarantees,,1731751137553.f3ed23ac403772e0fe4ca9ee892544ec.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/f3ed23ac403772e0fe4ca9ee892544ec/B/121e6e78f34e432d92caf3c87f619196, hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/f3ed23ac403772e0fe4ca9ee892544ec/B/6b4c1d88775e40b89a0762d4905b7d9f, hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/f3ed23ac403772e0fe4ca9ee892544ec/B/edee38773ced4b9ea467a4d5655b6334, hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/f3ed23ac403772e0fe4ca9ee892544ec/B/1ab050ebce744bc993da534cbe3617aa, hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/f3ed23ac403772e0fe4ca9ee892544ec/B/7b8eb25c363c4c66a4eb1bc0ba18a151, hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/f3ed23ac403772e0fe4ca9ee892544ec/B/2801d6506c4844e8ab66e1d8707f85ba, hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/f3ed23ac403772e0fe4ca9ee892544ec/B/e2460513c96644a9b59d2108e16c30c1, hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/f3ed23ac403772e0fe4ca9ee892544ec/B/0500168b3f534c849c2134d112232df9, hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/f3ed23ac403772e0fe4ca9ee892544ec/B/8cb982fa7b484471b2ab01d4d21174f0, hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/f3ed23ac403772e0fe4ca9ee892544ec/B/59e77647903f4960ababcd7f53bcaae7, hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/f3ed23ac403772e0fe4ca9ee892544ec/B/f889bfe6d6574832a2b04ec61ff06197, hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/f3ed23ac403772e0fe4ca9ee892544ec/B/192870787ef94f548016d8968e2641c5, hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/f3ed23ac403772e0fe4ca9ee892544ec/B/97437b43d51e496ba02e0a3029c174a6, hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/f3ed23ac403772e0fe4ca9ee892544ec/B/5c3c932b27c348f68a5ec53b7746f224, hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/f3ed23ac403772e0fe4ca9ee892544ec/B/44e03547f5ba453bb3ce37077e891369, hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/f3ed23ac403772e0fe4ca9ee892544ec/B/71c6cade454748a2ab9b75ee1c0575bc, hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/f3ed23ac403772e0fe4ca9ee892544ec/B/3df0f2796e9848a6af1729c89fb64a31, hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/f3ed23ac403772e0fe4ca9ee892544ec/B/198169af9b9a4f33b34b10f090bf7c75, hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/f3ed23ac403772e0fe4ca9ee892544ec/B/79248da02305417c88dd121613236b79, hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/f3ed23ac403772e0fe4ca9ee892544ec/B/5d60f3c9b044415cbf7981c5183581ba, hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/f3ed23ac403772e0fe4ca9ee892544ec/B/2dff7f545b324a0c9eb39bd60f22cc2a, hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/f3ed23ac403772e0fe4ca9ee892544ec/B/9c2850d44afa4fc7baca1f5aa115bb38, hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/f3ed23ac403772e0fe4ca9ee892544ec/B/921ee41b9b074ed2b0d2e2c28b2edec3, hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/f3ed23ac403772e0fe4ca9ee892544ec/B/e5ef6c31fd4d413bab0b528ff6c5762b, hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/f3ed23ac403772e0fe4ca9ee892544ec/B/7f75e570933e48349567b87f6b6576d1, hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/f3ed23ac403772e0fe4ca9ee892544ec/B/9020014cebb84c3d92e7decc88f8a37e] to archive 2024-11-16T09:59:22,428 DEBUG [StoreCloser-TestAcidGuarantees,,1731751137553.f3ed23ac403772e0fe4ca9ee892544ec.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-16T09:59:22,430 DEBUG [StoreCloser-TestAcidGuarantees,,1731751137553.f3ed23ac403772e0fe4ca9ee892544ec.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/f3ed23ac403772e0fe4ca9ee892544ec/B/121e6e78f34e432d92caf3c87f619196 to hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/archive/data/default/TestAcidGuarantees/f3ed23ac403772e0fe4ca9ee892544ec/B/121e6e78f34e432d92caf3c87f619196 2024-11-16T09:59:22,431 DEBUG [StoreCloser-TestAcidGuarantees,,1731751137553.f3ed23ac403772e0fe4ca9ee892544ec.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/f3ed23ac403772e0fe4ca9ee892544ec/B/6b4c1d88775e40b89a0762d4905b7d9f to hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/archive/data/default/TestAcidGuarantees/f3ed23ac403772e0fe4ca9ee892544ec/B/6b4c1d88775e40b89a0762d4905b7d9f 2024-11-16T09:59:22,431 DEBUG [StoreCloser-TestAcidGuarantees,,1731751137553.f3ed23ac403772e0fe4ca9ee892544ec.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/f3ed23ac403772e0fe4ca9ee892544ec/B/edee38773ced4b9ea467a4d5655b6334 to hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/archive/data/default/TestAcidGuarantees/f3ed23ac403772e0fe4ca9ee892544ec/B/edee38773ced4b9ea467a4d5655b6334 2024-11-16T09:59:22,432 DEBUG [StoreCloser-TestAcidGuarantees,,1731751137553.f3ed23ac403772e0fe4ca9ee892544ec.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/f3ed23ac403772e0fe4ca9ee892544ec/B/1ab050ebce744bc993da534cbe3617aa to hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/archive/data/default/TestAcidGuarantees/f3ed23ac403772e0fe4ca9ee892544ec/B/1ab050ebce744bc993da534cbe3617aa 2024-11-16T09:59:22,433 DEBUG [StoreCloser-TestAcidGuarantees,,1731751137553.f3ed23ac403772e0fe4ca9ee892544ec.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/f3ed23ac403772e0fe4ca9ee892544ec/B/7b8eb25c363c4c66a4eb1bc0ba18a151 to hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/archive/data/default/TestAcidGuarantees/f3ed23ac403772e0fe4ca9ee892544ec/B/7b8eb25c363c4c66a4eb1bc0ba18a151 2024-11-16T09:59:22,434 DEBUG [StoreCloser-TestAcidGuarantees,,1731751137553.f3ed23ac403772e0fe4ca9ee892544ec.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/f3ed23ac403772e0fe4ca9ee892544ec/B/2801d6506c4844e8ab66e1d8707f85ba to hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/archive/data/default/TestAcidGuarantees/f3ed23ac403772e0fe4ca9ee892544ec/B/2801d6506c4844e8ab66e1d8707f85ba 2024-11-16T09:59:22,435 DEBUG [StoreCloser-TestAcidGuarantees,,1731751137553.f3ed23ac403772e0fe4ca9ee892544ec.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/f3ed23ac403772e0fe4ca9ee892544ec/B/e2460513c96644a9b59d2108e16c30c1 to hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/archive/data/default/TestAcidGuarantees/f3ed23ac403772e0fe4ca9ee892544ec/B/e2460513c96644a9b59d2108e16c30c1 2024-11-16T09:59:22,436 DEBUG [StoreCloser-TestAcidGuarantees,,1731751137553.f3ed23ac403772e0fe4ca9ee892544ec.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/f3ed23ac403772e0fe4ca9ee892544ec/B/0500168b3f534c849c2134d112232df9 to hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/archive/data/default/TestAcidGuarantees/f3ed23ac403772e0fe4ca9ee892544ec/B/0500168b3f534c849c2134d112232df9 2024-11-16T09:59:22,437 DEBUG [StoreCloser-TestAcidGuarantees,,1731751137553.f3ed23ac403772e0fe4ca9ee892544ec.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/f3ed23ac403772e0fe4ca9ee892544ec/B/8cb982fa7b484471b2ab01d4d21174f0 to hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/archive/data/default/TestAcidGuarantees/f3ed23ac403772e0fe4ca9ee892544ec/B/8cb982fa7b484471b2ab01d4d21174f0 2024-11-16T09:59:22,437 DEBUG [StoreCloser-TestAcidGuarantees,,1731751137553.f3ed23ac403772e0fe4ca9ee892544ec.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/f3ed23ac403772e0fe4ca9ee892544ec/B/59e77647903f4960ababcd7f53bcaae7 to hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/archive/data/default/TestAcidGuarantees/f3ed23ac403772e0fe4ca9ee892544ec/B/59e77647903f4960ababcd7f53bcaae7 2024-11-16T09:59:22,438 DEBUG [StoreCloser-TestAcidGuarantees,,1731751137553.f3ed23ac403772e0fe4ca9ee892544ec.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/f3ed23ac403772e0fe4ca9ee892544ec/B/f889bfe6d6574832a2b04ec61ff06197 to hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/archive/data/default/TestAcidGuarantees/f3ed23ac403772e0fe4ca9ee892544ec/B/f889bfe6d6574832a2b04ec61ff06197 2024-11-16T09:59:22,439 DEBUG [StoreCloser-TestAcidGuarantees,,1731751137553.f3ed23ac403772e0fe4ca9ee892544ec.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/f3ed23ac403772e0fe4ca9ee892544ec/B/192870787ef94f548016d8968e2641c5 to hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/archive/data/default/TestAcidGuarantees/f3ed23ac403772e0fe4ca9ee892544ec/B/192870787ef94f548016d8968e2641c5 2024-11-16T09:59:22,440 DEBUG [StoreCloser-TestAcidGuarantees,,1731751137553.f3ed23ac403772e0fe4ca9ee892544ec.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/f3ed23ac403772e0fe4ca9ee892544ec/B/97437b43d51e496ba02e0a3029c174a6 to hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/archive/data/default/TestAcidGuarantees/f3ed23ac403772e0fe4ca9ee892544ec/B/97437b43d51e496ba02e0a3029c174a6 2024-11-16T09:59:22,441 DEBUG [StoreCloser-TestAcidGuarantees,,1731751137553.f3ed23ac403772e0fe4ca9ee892544ec.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/f3ed23ac403772e0fe4ca9ee892544ec/B/5c3c932b27c348f68a5ec53b7746f224 to hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/archive/data/default/TestAcidGuarantees/f3ed23ac403772e0fe4ca9ee892544ec/B/5c3c932b27c348f68a5ec53b7746f224 2024-11-16T09:59:22,441 DEBUG [StoreCloser-TestAcidGuarantees,,1731751137553.f3ed23ac403772e0fe4ca9ee892544ec.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/f3ed23ac403772e0fe4ca9ee892544ec/B/44e03547f5ba453bb3ce37077e891369 to hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/archive/data/default/TestAcidGuarantees/f3ed23ac403772e0fe4ca9ee892544ec/B/44e03547f5ba453bb3ce37077e891369 2024-11-16T09:59:22,442 DEBUG [StoreCloser-TestAcidGuarantees,,1731751137553.f3ed23ac403772e0fe4ca9ee892544ec.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/f3ed23ac403772e0fe4ca9ee892544ec/B/71c6cade454748a2ab9b75ee1c0575bc to hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/archive/data/default/TestAcidGuarantees/f3ed23ac403772e0fe4ca9ee892544ec/B/71c6cade454748a2ab9b75ee1c0575bc 2024-11-16T09:59:22,443 DEBUG [StoreCloser-TestAcidGuarantees,,1731751137553.f3ed23ac403772e0fe4ca9ee892544ec.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/f3ed23ac403772e0fe4ca9ee892544ec/B/3df0f2796e9848a6af1729c89fb64a31 to hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/archive/data/default/TestAcidGuarantees/f3ed23ac403772e0fe4ca9ee892544ec/B/3df0f2796e9848a6af1729c89fb64a31 2024-11-16T09:59:22,444 DEBUG [StoreCloser-TestAcidGuarantees,,1731751137553.f3ed23ac403772e0fe4ca9ee892544ec.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/f3ed23ac403772e0fe4ca9ee892544ec/B/198169af9b9a4f33b34b10f090bf7c75 to hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/archive/data/default/TestAcidGuarantees/f3ed23ac403772e0fe4ca9ee892544ec/B/198169af9b9a4f33b34b10f090bf7c75 2024-11-16T09:59:22,445 DEBUG [StoreCloser-TestAcidGuarantees,,1731751137553.f3ed23ac403772e0fe4ca9ee892544ec.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/f3ed23ac403772e0fe4ca9ee892544ec/B/79248da02305417c88dd121613236b79 to hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/archive/data/default/TestAcidGuarantees/f3ed23ac403772e0fe4ca9ee892544ec/B/79248da02305417c88dd121613236b79 2024-11-16T09:59:22,446 DEBUG [StoreCloser-TestAcidGuarantees,,1731751137553.f3ed23ac403772e0fe4ca9ee892544ec.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/f3ed23ac403772e0fe4ca9ee892544ec/B/5d60f3c9b044415cbf7981c5183581ba to hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/archive/data/default/TestAcidGuarantees/f3ed23ac403772e0fe4ca9ee892544ec/B/5d60f3c9b044415cbf7981c5183581ba 2024-11-16T09:59:22,446 DEBUG [StoreCloser-TestAcidGuarantees,,1731751137553.f3ed23ac403772e0fe4ca9ee892544ec.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/f3ed23ac403772e0fe4ca9ee892544ec/B/2dff7f545b324a0c9eb39bd60f22cc2a to hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/archive/data/default/TestAcidGuarantees/f3ed23ac403772e0fe4ca9ee892544ec/B/2dff7f545b324a0c9eb39bd60f22cc2a 2024-11-16T09:59:22,447 DEBUG [StoreCloser-TestAcidGuarantees,,1731751137553.f3ed23ac403772e0fe4ca9ee892544ec.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/f3ed23ac403772e0fe4ca9ee892544ec/B/9c2850d44afa4fc7baca1f5aa115bb38 to hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/archive/data/default/TestAcidGuarantees/f3ed23ac403772e0fe4ca9ee892544ec/B/9c2850d44afa4fc7baca1f5aa115bb38 2024-11-16T09:59:22,448 DEBUG [StoreCloser-TestAcidGuarantees,,1731751137553.f3ed23ac403772e0fe4ca9ee892544ec.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/f3ed23ac403772e0fe4ca9ee892544ec/B/921ee41b9b074ed2b0d2e2c28b2edec3 to hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/archive/data/default/TestAcidGuarantees/f3ed23ac403772e0fe4ca9ee892544ec/B/921ee41b9b074ed2b0d2e2c28b2edec3 2024-11-16T09:59:22,449 DEBUG [StoreCloser-TestAcidGuarantees,,1731751137553.f3ed23ac403772e0fe4ca9ee892544ec.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/f3ed23ac403772e0fe4ca9ee892544ec/B/e5ef6c31fd4d413bab0b528ff6c5762b to hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/archive/data/default/TestAcidGuarantees/f3ed23ac403772e0fe4ca9ee892544ec/B/e5ef6c31fd4d413bab0b528ff6c5762b 2024-11-16T09:59:22,450 DEBUG [StoreCloser-TestAcidGuarantees,,1731751137553.f3ed23ac403772e0fe4ca9ee892544ec.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/f3ed23ac403772e0fe4ca9ee892544ec/B/7f75e570933e48349567b87f6b6576d1 to hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/archive/data/default/TestAcidGuarantees/f3ed23ac403772e0fe4ca9ee892544ec/B/7f75e570933e48349567b87f6b6576d1 2024-11-16T09:59:22,451 DEBUG [StoreCloser-TestAcidGuarantees,,1731751137553.f3ed23ac403772e0fe4ca9ee892544ec.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/f3ed23ac403772e0fe4ca9ee892544ec/B/9020014cebb84c3d92e7decc88f8a37e to hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/archive/data/default/TestAcidGuarantees/f3ed23ac403772e0fe4ca9ee892544ec/B/9020014cebb84c3d92e7decc88f8a37e 2024-11-16T09:59:22,452 DEBUG [StoreCloser-TestAcidGuarantees,,1731751137553.f3ed23ac403772e0fe4ca9ee892544ec.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/f3ed23ac403772e0fe4ca9ee892544ec/C/f5e10ca7ce7f4643ac9f2b880087ce59, hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/f3ed23ac403772e0fe4ca9ee892544ec/C/03ddda3dad844fc09f3e33be2515e85e, hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/f3ed23ac403772e0fe4ca9ee892544ec/C/b747dafa9b994f2ba6ec3cb9ada2e63d, hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/f3ed23ac403772e0fe4ca9ee892544ec/C/4336fc254c134e0b93fa1380900fa45f, hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/f3ed23ac403772e0fe4ca9ee892544ec/C/b20aebb0d817468f8d6fb345deac0dc9, hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/f3ed23ac403772e0fe4ca9ee892544ec/C/7a98e6e6c80d4413a5a5f621e7789567, hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/f3ed23ac403772e0fe4ca9ee892544ec/C/3a6d4b5ddb9a453da1011ee0b36a82b7, hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/f3ed23ac403772e0fe4ca9ee892544ec/C/1003ab34c2204b9da43a22b69b58c635, hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/f3ed23ac403772e0fe4ca9ee892544ec/C/3349459d23bd43dd85817bbd6c9715fd, hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/f3ed23ac403772e0fe4ca9ee892544ec/C/98ad7c5b73d9470bbe63c231e0379a5a, hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/f3ed23ac403772e0fe4ca9ee892544ec/C/c828236206584489b0b4b191772c331b, hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/f3ed23ac403772e0fe4ca9ee892544ec/C/515f53734f6e4430b06393764cb56c0f, hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/f3ed23ac403772e0fe4ca9ee892544ec/C/a4edb180540e4bb7a2083c0f6df737f7, hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/f3ed23ac403772e0fe4ca9ee892544ec/C/cf880f53d4a14e80999862e85f6b78ee, hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/f3ed23ac403772e0fe4ca9ee892544ec/C/c7691ed6881640f39f20fbf39543222c, hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/f3ed23ac403772e0fe4ca9ee892544ec/C/e7becd72ceb849a3b38a042102b3a7a0, hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/f3ed23ac403772e0fe4ca9ee892544ec/C/1278f4f12e514b53ba17c59c2c5fa75c, hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/f3ed23ac403772e0fe4ca9ee892544ec/C/d35b1dfd46a04514a8e15ae0d6e3ba67, hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/f3ed23ac403772e0fe4ca9ee892544ec/C/04b64f8be1d64da9880fa2578af827ac, hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/f3ed23ac403772e0fe4ca9ee892544ec/C/42f37ce77c114494ab1505a0d320671a, hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/f3ed23ac403772e0fe4ca9ee892544ec/C/246486ae295b4ce6b3d0df8dfcebfe51, hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/f3ed23ac403772e0fe4ca9ee892544ec/C/9b15a7dac9a24c048f9b79c39fbc2aad, hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/f3ed23ac403772e0fe4ca9ee892544ec/C/24ccf25c8a5a4d6e8f0d6ebdf95b6954, hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/f3ed23ac403772e0fe4ca9ee892544ec/C/76b2417e63f04287845faacef82c7350] to archive 2024-11-16T09:59:22,453 DEBUG [StoreCloser-TestAcidGuarantees,,1731751137553.f3ed23ac403772e0fe4ca9ee892544ec.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-16T09:59:22,454 DEBUG [StoreCloser-TestAcidGuarantees,,1731751137553.f3ed23ac403772e0fe4ca9ee892544ec.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/f3ed23ac403772e0fe4ca9ee892544ec/C/f5e10ca7ce7f4643ac9f2b880087ce59 to hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/archive/data/default/TestAcidGuarantees/f3ed23ac403772e0fe4ca9ee892544ec/C/f5e10ca7ce7f4643ac9f2b880087ce59 2024-11-16T09:59:22,455 DEBUG [StoreCloser-TestAcidGuarantees,,1731751137553.f3ed23ac403772e0fe4ca9ee892544ec.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/f3ed23ac403772e0fe4ca9ee892544ec/C/03ddda3dad844fc09f3e33be2515e85e to hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/archive/data/default/TestAcidGuarantees/f3ed23ac403772e0fe4ca9ee892544ec/C/03ddda3dad844fc09f3e33be2515e85e 2024-11-16T09:59:22,456 DEBUG [StoreCloser-TestAcidGuarantees,,1731751137553.f3ed23ac403772e0fe4ca9ee892544ec.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/f3ed23ac403772e0fe4ca9ee892544ec/C/b747dafa9b994f2ba6ec3cb9ada2e63d to hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/archive/data/default/TestAcidGuarantees/f3ed23ac403772e0fe4ca9ee892544ec/C/b747dafa9b994f2ba6ec3cb9ada2e63d 2024-11-16T09:59:22,457 DEBUG [StoreCloser-TestAcidGuarantees,,1731751137553.f3ed23ac403772e0fe4ca9ee892544ec.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/f3ed23ac403772e0fe4ca9ee892544ec/C/4336fc254c134e0b93fa1380900fa45f to hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/archive/data/default/TestAcidGuarantees/f3ed23ac403772e0fe4ca9ee892544ec/C/4336fc254c134e0b93fa1380900fa45f 2024-11-16T09:59:22,458 DEBUG [StoreCloser-TestAcidGuarantees,,1731751137553.f3ed23ac403772e0fe4ca9ee892544ec.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/f3ed23ac403772e0fe4ca9ee892544ec/C/b20aebb0d817468f8d6fb345deac0dc9 to hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/archive/data/default/TestAcidGuarantees/f3ed23ac403772e0fe4ca9ee892544ec/C/b20aebb0d817468f8d6fb345deac0dc9 2024-11-16T09:59:22,458 DEBUG [StoreCloser-TestAcidGuarantees,,1731751137553.f3ed23ac403772e0fe4ca9ee892544ec.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/f3ed23ac403772e0fe4ca9ee892544ec/C/7a98e6e6c80d4413a5a5f621e7789567 to hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/archive/data/default/TestAcidGuarantees/f3ed23ac403772e0fe4ca9ee892544ec/C/7a98e6e6c80d4413a5a5f621e7789567 2024-11-16T09:59:22,459 DEBUG [StoreCloser-TestAcidGuarantees,,1731751137553.f3ed23ac403772e0fe4ca9ee892544ec.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/f3ed23ac403772e0fe4ca9ee892544ec/C/3a6d4b5ddb9a453da1011ee0b36a82b7 to hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/archive/data/default/TestAcidGuarantees/f3ed23ac403772e0fe4ca9ee892544ec/C/3a6d4b5ddb9a453da1011ee0b36a82b7 2024-11-16T09:59:22,460 DEBUG [StoreCloser-TestAcidGuarantees,,1731751137553.f3ed23ac403772e0fe4ca9ee892544ec.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/f3ed23ac403772e0fe4ca9ee892544ec/C/1003ab34c2204b9da43a22b69b58c635 to hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/archive/data/default/TestAcidGuarantees/f3ed23ac403772e0fe4ca9ee892544ec/C/1003ab34c2204b9da43a22b69b58c635 2024-11-16T09:59:22,461 DEBUG [StoreCloser-TestAcidGuarantees,,1731751137553.f3ed23ac403772e0fe4ca9ee892544ec.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/f3ed23ac403772e0fe4ca9ee892544ec/C/3349459d23bd43dd85817bbd6c9715fd to hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/archive/data/default/TestAcidGuarantees/f3ed23ac403772e0fe4ca9ee892544ec/C/3349459d23bd43dd85817bbd6c9715fd 2024-11-16T09:59:22,462 DEBUG [StoreCloser-TestAcidGuarantees,,1731751137553.f3ed23ac403772e0fe4ca9ee892544ec.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/f3ed23ac403772e0fe4ca9ee892544ec/C/98ad7c5b73d9470bbe63c231e0379a5a to hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/archive/data/default/TestAcidGuarantees/f3ed23ac403772e0fe4ca9ee892544ec/C/98ad7c5b73d9470bbe63c231e0379a5a 2024-11-16T09:59:22,463 DEBUG [StoreCloser-TestAcidGuarantees,,1731751137553.f3ed23ac403772e0fe4ca9ee892544ec.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/f3ed23ac403772e0fe4ca9ee892544ec/C/c828236206584489b0b4b191772c331b to hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/archive/data/default/TestAcidGuarantees/f3ed23ac403772e0fe4ca9ee892544ec/C/c828236206584489b0b4b191772c331b 2024-11-16T09:59:22,463 DEBUG [StoreCloser-TestAcidGuarantees,,1731751137553.f3ed23ac403772e0fe4ca9ee892544ec.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/f3ed23ac403772e0fe4ca9ee892544ec/C/515f53734f6e4430b06393764cb56c0f to hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/archive/data/default/TestAcidGuarantees/f3ed23ac403772e0fe4ca9ee892544ec/C/515f53734f6e4430b06393764cb56c0f 2024-11-16T09:59:22,464 DEBUG [StoreCloser-TestAcidGuarantees,,1731751137553.f3ed23ac403772e0fe4ca9ee892544ec.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/f3ed23ac403772e0fe4ca9ee892544ec/C/a4edb180540e4bb7a2083c0f6df737f7 to hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/archive/data/default/TestAcidGuarantees/f3ed23ac403772e0fe4ca9ee892544ec/C/a4edb180540e4bb7a2083c0f6df737f7 2024-11-16T09:59:22,465 DEBUG [StoreCloser-TestAcidGuarantees,,1731751137553.f3ed23ac403772e0fe4ca9ee892544ec.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/f3ed23ac403772e0fe4ca9ee892544ec/C/cf880f53d4a14e80999862e85f6b78ee to hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/archive/data/default/TestAcidGuarantees/f3ed23ac403772e0fe4ca9ee892544ec/C/cf880f53d4a14e80999862e85f6b78ee 2024-11-16T09:59:22,466 DEBUG [StoreCloser-TestAcidGuarantees,,1731751137553.f3ed23ac403772e0fe4ca9ee892544ec.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/f3ed23ac403772e0fe4ca9ee892544ec/C/c7691ed6881640f39f20fbf39543222c to hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/archive/data/default/TestAcidGuarantees/f3ed23ac403772e0fe4ca9ee892544ec/C/c7691ed6881640f39f20fbf39543222c 2024-11-16T09:59:22,467 DEBUG [StoreCloser-TestAcidGuarantees,,1731751137553.f3ed23ac403772e0fe4ca9ee892544ec.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/f3ed23ac403772e0fe4ca9ee892544ec/C/e7becd72ceb849a3b38a042102b3a7a0 to hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/archive/data/default/TestAcidGuarantees/f3ed23ac403772e0fe4ca9ee892544ec/C/e7becd72ceb849a3b38a042102b3a7a0 2024-11-16T09:59:22,467 DEBUG [StoreCloser-TestAcidGuarantees,,1731751137553.f3ed23ac403772e0fe4ca9ee892544ec.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/f3ed23ac403772e0fe4ca9ee892544ec/C/1278f4f12e514b53ba17c59c2c5fa75c to hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/archive/data/default/TestAcidGuarantees/f3ed23ac403772e0fe4ca9ee892544ec/C/1278f4f12e514b53ba17c59c2c5fa75c 2024-11-16T09:59:22,468 DEBUG [StoreCloser-TestAcidGuarantees,,1731751137553.f3ed23ac403772e0fe4ca9ee892544ec.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/f3ed23ac403772e0fe4ca9ee892544ec/C/d35b1dfd46a04514a8e15ae0d6e3ba67 to hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/archive/data/default/TestAcidGuarantees/f3ed23ac403772e0fe4ca9ee892544ec/C/d35b1dfd46a04514a8e15ae0d6e3ba67 2024-11-16T09:59:22,469 DEBUG [StoreCloser-TestAcidGuarantees,,1731751137553.f3ed23ac403772e0fe4ca9ee892544ec.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/f3ed23ac403772e0fe4ca9ee892544ec/C/04b64f8be1d64da9880fa2578af827ac to hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/archive/data/default/TestAcidGuarantees/f3ed23ac403772e0fe4ca9ee892544ec/C/04b64f8be1d64da9880fa2578af827ac 2024-11-16T09:59:22,470 DEBUG [StoreCloser-TestAcidGuarantees,,1731751137553.f3ed23ac403772e0fe4ca9ee892544ec.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/f3ed23ac403772e0fe4ca9ee892544ec/C/42f37ce77c114494ab1505a0d320671a to hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/archive/data/default/TestAcidGuarantees/f3ed23ac403772e0fe4ca9ee892544ec/C/42f37ce77c114494ab1505a0d320671a 2024-11-16T09:59:22,471 DEBUG [StoreCloser-TestAcidGuarantees,,1731751137553.f3ed23ac403772e0fe4ca9ee892544ec.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/f3ed23ac403772e0fe4ca9ee892544ec/C/246486ae295b4ce6b3d0df8dfcebfe51 to hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/archive/data/default/TestAcidGuarantees/f3ed23ac403772e0fe4ca9ee892544ec/C/246486ae295b4ce6b3d0df8dfcebfe51 2024-11-16T09:59:22,471 DEBUG [StoreCloser-TestAcidGuarantees,,1731751137553.f3ed23ac403772e0fe4ca9ee892544ec.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/f3ed23ac403772e0fe4ca9ee892544ec/C/9b15a7dac9a24c048f9b79c39fbc2aad to hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/archive/data/default/TestAcidGuarantees/f3ed23ac403772e0fe4ca9ee892544ec/C/9b15a7dac9a24c048f9b79c39fbc2aad 2024-11-16T09:59:22,473 DEBUG [StoreCloser-TestAcidGuarantees,,1731751137553.f3ed23ac403772e0fe4ca9ee892544ec.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/f3ed23ac403772e0fe4ca9ee892544ec/C/24ccf25c8a5a4d6e8f0d6ebdf95b6954 to hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/archive/data/default/TestAcidGuarantees/f3ed23ac403772e0fe4ca9ee892544ec/C/24ccf25c8a5a4d6e8f0d6ebdf95b6954 2024-11-16T09:59:22,473 DEBUG [StoreCloser-TestAcidGuarantees,,1731751137553.f3ed23ac403772e0fe4ca9ee892544ec.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/f3ed23ac403772e0fe4ca9ee892544ec/C/76b2417e63f04287845faacef82c7350 to hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/archive/data/default/TestAcidGuarantees/f3ed23ac403772e0fe4ca9ee892544ec/C/76b2417e63f04287845faacef82c7350 2024-11-16T09:59:22,477 DEBUG [RS_CLOSE_REGION-regionserver/037c58a19ddb:0-0 {event_type=M_RS_CLOSE_REGION, pid=87}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/f3ed23ac403772e0fe4ca9ee892544ec/recovered.edits/418.seqid, newMaxSeqId=418, maxSeqId=1 2024-11-16T09:59:22,478 INFO [RS_CLOSE_REGION-regionserver/037c58a19ddb:0-0 {event_type=M_RS_CLOSE_REGION, pid=87}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1731751137553.f3ed23ac403772e0fe4ca9ee892544ec. 2024-11-16T09:59:22,478 DEBUG [RS_CLOSE_REGION-regionserver/037c58a19ddb:0-0 {event_type=M_RS_CLOSE_REGION, pid=87}] regionserver.HRegion(1676): Region close journal for f3ed23ac403772e0fe4ca9ee892544ec: Waiting for close lock at 1731751160535Running coprocessor pre-close hooks at 1731751160535Disabling compacts and flushes for region at 1731751160535Disabling writes for close at 1731751161157 (+622 ms)Obtaining lock to block concurrent updates at 1731751161157Preparing flush snapshotting stores in f3ed23ac403772e0fe4ca9ee892544ec at 1731751161157Finished memstore snapshotting TestAcidGuarantees,,1731751137553.f3ed23ac403772e0fe4ca9ee892544ec., syncing WAL and waiting on mvcc, flushsize=dataSize=20610, getHeapSize=54720, getOffHeapSize=0, getCellsCount=450 at 1731751161158 (+1 ms)Flushing stores of TestAcidGuarantees,,1731751137553.f3ed23ac403772e0fe4ca9ee892544ec. at 1731751161158Flushing f3ed23ac403772e0fe4ca9ee892544ec/A: creating writer at 1731751161158Flushing f3ed23ac403772e0fe4ca9ee892544ec/A: appending metadata at 1731751161160 (+2 ms)Flushing f3ed23ac403772e0fe4ca9ee892544ec/A: closing flushed file at 1731751161160Flushing f3ed23ac403772e0fe4ca9ee892544ec/B: creating writer at 1731751161568 (+408 ms)Flushing f3ed23ac403772e0fe4ca9ee892544ec/B: appending metadata at 1731751161570 (+2 ms)Flushing f3ed23ac403772e0fe4ca9ee892544ec/B: closing flushed file at 1731751161570Flushing f3ed23ac403772e0fe4ca9ee892544ec/C: creating writer at 1731751161979 (+409 ms)Flushing f3ed23ac403772e0fe4ca9ee892544ec/C: appending metadata at 1731751161981 (+2 ms)Flushing f3ed23ac403772e0fe4ca9ee892544ec/C: closing flushed file at 1731751161981Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@ef6baec: reopening flushed file at 1731751162389 (+408 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@745e862: reopening flushed file at 1731751162393 (+4 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@447fa980: reopening flushed file at 1731751162397 (+4 ms)Finished flush of dataSize ~20.13 KB/20610, heapSize ~53.44 KB/54720, currentSize=0 B/0 for f3ed23ac403772e0fe4ca9ee892544ec in 1244ms, sequenceid=415, compaction requested=true at 1731751162401 (+4 ms)Writing region close event to WAL at 1731751162475 (+74 ms)Running coprocessor post-close hooks at 1731751162478 (+3 ms)Closed at 1731751162478 2024-11-16T09:59:22,479 INFO [RS_CLOSE_REGION-regionserver/037c58a19ddb:0-0 {event_type=M_RS_CLOSE_REGION, pid=87}] handler.UnassignRegionHandler(157): Closed f3ed23ac403772e0fe4ca9ee892544ec 2024-11-16T09:59:22,480 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=86 updating hbase:meta row=f3ed23ac403772e0fe4ca9ee892544ec, regionState=CLOSED 2024-11-16T09:59:22,481 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=87, ppid=86, state=RUNNABLE, hasLock=false; CloseRegionProcedure f3ed23ac403772e0fe4ca9ee892544ec, server=037c58a19ddb,37267,1731751077844 because future has completed 2024-11-16T09:59:22,483 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=87, resume processing ppid=86 2024-11-16T09:59:22,483 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=87, ppid=86, state=SUCCESS, hasLock=false; CloseRegionProcedure f3ed23ac403772e0fe4ca9ee892544ec, server=037c58a19ddb,37267,1731751077844 in 2.0990 sec 2024-11-16T09:59:22,484 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=86, resume processing ppid=85 2024-11-16T09:59:22,485 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=86, ppid=85, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=f3ed23ac403772e0fe4ca9ee892544ec, UNASSIGN in 2.1030 sec 2024-11-16T09:59:22,486 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=85, resume processing ppid=84 2024-11-16T09:59:22,486 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=85, ppid=84, state=SUCCESS, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees in 2.1060 sec 2024-11-16T09:59:22,488 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731751162488"}]},"ts":"1731751162488"} 2024-11-16T09:59:22,489 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLED in hbase:meta 2024-11-16T09:59:22,489 INFO [PEWorker-4 {}] procedure.DisableTableProcedure(296): Set TestAcidGuarantees to state=DISABLED 2024-11-16T09:59:22,491 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=84, state=SUCCESS, hasLock=false; DisableTableProcedure table=TestAcidGuarantees in 2.1150 sec 2024-11-16T09:59:22,533 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37877 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=84 2024-11-16T09:59:22,534 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DISABLE, Table Name: default:TestAcidGuarantees completed 2024-11-16T09:59:22,534 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37877 {}] master.HMaster$5(2570): Client=jenkins//172.17.0.2 delete TestAcidGuarantees 2024-11-16T09:59:22,535 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37877 {}] procedure2.ProcedureExecutor(1139): Stored pid=88, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees 2024-11-16T09:59:22,536 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(103): Waiting for RIT for pid=88, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-16T09:59:22,537 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37877 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=88 2024-11-16T09:59:22,537 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(115): Deleting regions from filesystem for pid=88, state=RUNNABLE:DELETE_TABLE_CLEAR_FS_LAYOUT, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-16T09:59:22,540 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(131): ARCHIVING hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/f3ed23ac403772e0fe4ca9ee892544ec 2024-11-16T09:59:22,542 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(159): Archiving [FileablePath, hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/f3ed23ac403772e0fe4ca9ee892544ec/A, FileablePath, hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/f3ed23ac403772e0fe4ca9ee892544ec/B, FileablePath, hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/f3ed23ac403772e0fe4ca9ee892544ec/C, FileablePath, hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/f3ed23ac403772e0fe4ca9ee892544ec/recovered.edits] 2024-11-16T09:59:22,544 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/f3ed23ac403772e0fe4ca9ee892544ec/A/a2b10ea75ac040618f8632c5bc0636d4 to hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/archive/data/default/TestAcidGuarantees/f3ed23ac403772e0fe4ca9ee892544ec/A/a2b10ea75ac040618f8632c5bc0636d4 2024-11-16T09:59:22,545 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/f3ed23ac403772e0fe4ca9ee892544ec/A/c80e5c0c86dc40869a7ee85491ab01c0 to hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/archive/data/default/TestAcidGuarantees/f3ed23ac403772e0fe4ca9ee892544ec/A/c80e5c0c86dc40869a7ee85491ab01c0 2024-11-16T09:59:22,546 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/f3ed23ac403772e0fe4ca9ee892544ec/A/e1ba74d1fb694d7ca6d71ad6778d6ac6 to hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/archive/data/default/TestAcidGuarantees/f3ed23ac403772e0fe4ca9ee892544ec/A/e1ba74d1fb694d7ca6d71ad6778d6ac6 2024-11-16T09:59:22,547 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/f3ed23ac403772e0fe4ca9ee892544ec/A/f23a43b0b66f42ef8081f7a02159bd86 to hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/archive/data/default/TestAcidGuarantees/f3ed23ac403772e0fe4ca9ee892544ec/A/f23a43b0b66f42ef8081f7a02159bd86 2024-11-16T09:59:22,549 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/f3ed23ac403772e0fe4ca9ee892544ec/B/2da0bbf876be43f8828f0312f724aada to hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/archive/data/default/TestAcidGuarantees/f3ed23ac403772e0fe4ca9ee892544ec/B/2da0bbf876be43f8828f0312f724aada 2024-11-16T09:59:22,550 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/f3ed23ac403772e0fe4ca9ee892544ec/B/651328a01a444b6d81fe1cf63276890b to hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/archive/data/default/TestAcidGuarantees/f3ed23ac403772e0fe4ca9ee892544ec/B/651328a01a444b6d81fe1cf63276890b 2024-11-16T09:59:22,551 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/f3ed23ac403772e0fe4ca9ee892544ec/B/a68d4c6cd7d542f7b49c6bc68a97ed7e to hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/archive/data/default/TestAcidGuarantees/f3ed23ac403772e0fe4ca9ee892544ec/B/a68d4c6cd7d542f7b49c6bc68a97ed7e 2024-11-16T09:59:22,552 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/f3ed23ac403772e0fe4ca9ee892544ec/B/b56e0226632242a8b999c8e3e32b7f63 to hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/archive/data/default/TestAcidGuarantees/f3ed23ac403772e0fe4ca9ee892544ec/B/b56e0226632242a8b999c8e3e32b7f63 2024-11-16T09:59:22,553 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/f3ed23ac403772e0fe4ca9ee892544ec/C/03d27bbfa32d4a2d8ec7125bef3bb47c to hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/archive/data/default/TestAcidGuarantees/f3ed23ac403772e0fe4ca9ee892544ec/C/03d27bbfa32d4a2d8ec7125bef3bb47c 2024-11-16T09:59:22,554 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/f3ed23ac403772e0fe4ca9ee892544ec/C/5974b10c166b4ed6ac601358bb28f686 to hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/archive/data/default/TestAcidGuarantees/f3ed23ac403772e0fe4ca9ee892544ec/C/5974b10c166b4ed6ac601358bb28f686 2024-11-16T09:59:22,555 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/f3ed23ac403772e0fe4ca9ee892544ec/C/5c78ce09ea8c45479341bc9aeaa9aece to hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/archive/data/default/TestAcidGuarantees/f3ed23ac403772e0fe4ca9ee892544ec/C/5c78ce09ea8c45479341bc9aeaa9aece 2024-11-16T09:59:22,556 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/f3ed23ac403772e0fe4ca9ee892544ec/C/c97ad2e6a0da44c2ba953469d1ef4f35 to hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/archive/data/default/TestAcidGuarantees/f3ed23ac403772e0fe4ca9ee892544ec/C/c97ad2e6a0da44c2ba953469d1ef4f35 2024-11-16T09:59:22,558 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/f3ed23ac403772e0fe4ca9ee892544ec/C/cec8fedd185343628fdb34006411b43d to hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/archive/data/default/TestAcidGuarantees/f3ed23ac403772e0fe4ca9ee892544ec/C/cec8fedd185343628fdb34006411b43d 2024-11-16T09:59:22,560 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/f3ed23ac403772e0fe4ca9ee892544ec/recovered.edits/418.seqid to hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/archive/data/default/TestAcidGuarantees/f3ed23ac403772e0fe4ca9ee892544ec/recovered.edits/418.seqid 2024-11-16T09:59:22,560 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(610): Deleted hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/f3ed23ac403772e0fe4ca9ee892544ec 2024-11-16T09:59:22,560 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(313): Archived TestAcidGuarantees regions 2024-11-16T09:59:22,562 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(120): Deleting regions from META for pid=88, state=RUNNABLE:DELETE_TABLE_REMOVE_FROM_META, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-16T09:59:22,565 WARN [PEWorker-5 {}] procedure.DeleteTableProcedure(370): Deleting some vestigial 1 rows of TestAcidGuarantees from hbase:meta 2024-11-16T09:59:22,567 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(407): Removing 'TestAcidGuarantees' descriptor. 2024-11-16T09:59:22,568 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(126): Deleting assignment state for pid=88, state=RUNNABLE:DELETE_TABLE_UNASSIGN_REGIONS, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-16T09:59:22,568 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(397): Removing 'TestAcidGuarantees' from region states. 2024-11-16T09:59:22,568 DEBUG [PEWorker-5 {}] assignment.RegionStateStore(723): Delete {"totalColumns":1,"row":"TestAcidGuarantees,,1731751137553.f3ed23ac403772e0fe4ca9ee892544ec.","families":{"info":[{"qualifier":"","vlen":0,"tag":[],"timestamp":"1731751162568"}]},"ts":"9223372036854775807"} 2024-11-16T09:59:22,570 INFO [PEWorker-5 {}] assignment.RegionStateStore(562): Deleted 1 regions from META 2024-11-16T09:59:22,570 DEBUG [PEWorker-5 {}] assignment.RegionStateStore(563): Deleted regions: [{ENCODED => f3ed23ac403772e0fe4ca9ee892544ec, NAME => 'TestAcidGuarantees,,1731751137553.f3ed23ac403772e0fe4ca9ee892544ec.', STARTKEY => '', ENDKEY => ''}] 2024-11-16T09:59:22,570 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(401): Marking 'TestAcidGuarantees' as deleted. 2024-11-16T09:59:22,570 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(964): Delete {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":0,"tag":[],"timestamp":"1731751162570"}]},"ts":"9223372036854775807"} 2024-11-16T09:59:22,572 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(867): Deleted table TestAcidGuarantees state from META 2024-11-16T09:59:22,572 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(133): Finished pid=88, state=RUNNABLE:DELETE_TABLE_POST_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-16T09:59:22,573 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=88, state=SUCCESS, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees in 38 msec 2024-11-16T09:59:22,642 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37877 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=88 2024-11-16T09:59:22,643 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocator(219): Clear meta cache for TestAcidGuarantees 2024-11-16T09:59:22,643 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DELETE, Table Name: default:TestAcidGuarantees completed 2024-11-16T09:59:22,652 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: TestAcidGuaranteesWithNoInMemCompaction#testGetAtomicity Thread=241 (was 243), OpenFileDescriptor=457 (was 463), MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=361 (was 357) - SystemLoadAverage LEAK? -, ProcessCount=11 (was 11), AvailableMemoryMB=5995 (was 6047) 2024-11-16T09:59:22,661 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: TestAcidGuaranteesWithNoInMemCompaction#testMobScanAtomicity Thread=241, OpenFileDescriptor=457, MaxFileDescriptor=1048576, SystemLoadAverage=361, ProcessCount=11, AvailableMemoryMB=5995 2024-11-16T09:59:22,662 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37877 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (131072) is too small, which might cause very frequent flushing. 2024-11-16T09:59:22,662 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37877 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.2 create 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'NONE'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-16T09:59:22,663 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37877 {}] procedure2.ProcedureExecutor(1139): Stored pid=89, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=TestAcidGuarantees 2024-11-16T09:59:22,664 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=89, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_PRE_OPERATION 2024-11-16T09:59:22,664 DEBUG [PEWorker-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T09:59:22,664 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37877 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.2 procedure request for creating table: namespace: "default" qualifier: "TestAcidGuarantees" procId is: 89 2024-11-16T09:59:22,665 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37877 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=89 2024-11-16T09:59:22,665 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=89, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-11-16T09:59:22,669 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37591 is added to blk_1073742168_1344 (size=959) 2024-11-16T09:59:22,772 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37877 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=89 2024-11-16T09:59:22,983 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37877 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=89 2024-11-16T09:59:23,071 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => ffe1934c280093c181c2b062a05e0908, NAME => 'TestAcidGuarantees,,1731751162662.ffe1934c280093c181c2b062a05e0908.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'NONE', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592 2024-11-16T09:59:23,076 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37591 is added to blk_1073742169_1345 (size=53) 2024-11-16T09:59:23,293 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37877 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=89 2024-11-16T09:59:23,477 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1731751162662.ffe1934c280093c181c2b062a05e0908.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-16T09:59:23,477 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1722): Closing ffe1934c280093c181c2b062a05e0908, disabling compactions & flushes 2024-11-16T09:59:23,477 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1731751162662.ffe1934c280093c181c2b062a05e0908. 2024-11-16T09:59:23,477 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1731751162662.ffe1934c280093c181c2b062a05e0908. 2024-11-16T09:59:23,477 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1731751162662.ffe1934c280093c181c2b062a05e0908. after waiting 0 ms 2024-11-16T09:59:23,477 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1731751162662.ffe1934c280093c181c2b062a05e0908. 2024-11-16T09:59:23,477 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1731751162662.ffe1934c280093c181c2b062a05e0908. 2024-11-16T09:59:23,477 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1676): Region close journal for ffe1934c280093c181c2b062a05e0908: Waiting for close lock at 1731751163477Disabling compacts and flushes for region at 1731751163477Disabling writes for close at 1731751163477Writing region close event to WAL at 1731751163477Closed at 1731751163477 2024-11-16T09:59:23,478 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=89, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ADD_TO_META 2024-11-16T09:59:23,479 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"TestAcidGuarantees,,1731751162662.ffe1934c280093c181c2b062a05e0908.","families":{"info":[{"qualifier":"regioninfo","vlen":52,"tag":[],"timestamp":"1731751163479"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1731751163479"}]},"ts":"1731751163479"} 2024-11-16T09:59:23,480 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-11-16T09:59:23,481 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=89, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-11-16T09:59:23,481 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731751163481"}]},"ts":"1731751163481"} 2024-11-16T09:59:23,482 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLING in hbase:meta 2024-11-16T09:59:23,482 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=90, ppid=89, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=ffe1934c280093c181c2b062a05e0908, ASSIGN}] 2024-11-16T09:59:23,483 INFO [PEWorker-3 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=90, ppid=89, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=ffe1934c280093c181c2b062a05e0908, ASSIGN 2024-11-16T09:59:23,484 INFO [PEWorker-3 {}] assignment.TransitRegionStateProcedure(269): Starting pid=90, ppid=89, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=ffe1934c280093c181c2b062a05e0908, ASSIGN; state=OFFLINE, location=037c58a19ddb,37267,1731751077844; forceNewPlan=false, retain=false 2024-11-16T09:59:23,634 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=90 updating hbase:meta row=ffe1934c280093c181c2b062a05e0908, regionState=OPENING, regionLocation=037c58a19ddb,37267,1731751077844 2024-11-16T09:59:23,636 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=90, ppid=89, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=ffe1934c280093c181c2b062a05e0908, ASSIGN because future has completed 2024-11-16T09:59:23,636 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=91, ppid=90, state=RUNNABLE, hasLock=false; OpenRegionProcedure ffe1934c280093c181c2b062a05e0908, server=037c58a19ddb,37267,1731751077844}] 2024-11-16T09:59:23,791 INFO [RS_OPEN_REGION-regionserver/037c58a19ddb:0-0 {event_type=M_RS_OPEN_REGION, pid=91}] handler.AssignRegionHandler(132): Open TestAcidGuarantees,,1731751162662.ffe1934c280093c181c2b062a05e0908. 2024-11-16T09:59:23,791 DEBUG [RS_OPEN_REGION-regionserver/037c58a19ddb:0-0 {event_type=M_RS_OPEN_REGION, pid=91}] regionserver.HRegion(7752): Opening region: {ENCODED => ffe1934c280093c181c2b062a05e0908, NAME => 'TestAcidGuarantees,,1731751162662.ffe1934c280093c181c2b062a05e0908.', STARTKEY => '', ENDKEY => ''} 2024-11-16T09:59:23,791 DEBUG [RS_OPEN_REGION-regionserver/037c58a19ddb:0-0 {event_type=M_RS_OPEN_REGION, pid=91}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestAcidGuarantees ffe1934c280093c181c2b062a05e0908 2024-11-16T09:59:23,792 DEBUG [RS_OPEN_REGION-regionserver/037c58a19ddb:0-0 {event_type=M_RS_OPEN_REGION, pid=91}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1731751162662.ffe1934c280093c181c2b062a05e0908.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-16T09:59:23,792 DEBUG [RS_OPEN_REGION-regionserver/037c58a19ddb:0-0 {event_type=M_RS_OPEN_REGION, pid=91}] regionserver.HRegion(7794): checking encryption for ffe1934c280093c181c2b062a05e0908 2024-11-16T09:59:23,792 DEBUG [RS_OPEN_REGION-regionserver/037c58a19ddb:0-0 {event_type=M_RS_OPEN_REGION, pid=91}] regionserver.HRegion(7797): checking classloading for ffe1934c280093c181c2b062a05e0908 2024-11-16T09:59:23,793 INFO [StoreOpener-ffe1934c280093c181c2b062a05e0908-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family A of region ffe1934c280093c181c2b062a05e0908 2024-11-16T09:59:23,794 INFO [StoreOpener-ffe1934c280093c181c2b062a05e0908-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region ffe1934c280093c181c2b062a05e0908 columnFamilyName A 2024-11-16T09:59:23,794 DEBUG [StoreOpener-ffe1934c280093c181c2b062a05e0908-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T09:59:23,794 INFO [StoreOpener-ffe1934c280093c181c2b062a05e0908-1 {}] regionserver.HStore(327): Store=ffe1934c280093c181c2b062a05e0908/A, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-16T09:59:23,795 INFO [StoreOpener-ffe1934c280093c181c2b062a05e0908-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family B of region ffe1934c280093c181c2b062a05e0908 2024-11-16T09:59:23,795 INFO [StoreOpener-ffe1934c280093c181c2b062a05e0908-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region ffe1934c280093c181c2b062a05e0908 columnFamilyName B 2024-11-16T09:59:23,795 DEBUG [StoreOpener-ffe1934c280093c181c2b062a05e0908-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T09:59:23,796 INFO [StoreOpener-ffe1934c280093c181c2b062a05e0908-1 {}] regionserver.HStore(327): Store=ffe1934c280093c181c2b062a05e0908/B, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-16T09:59:23,796 INFO [StoreOpener-ffe1934c280093c181c2b062a05e0908-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family C of region ffe1934c280093c181c2b062a05e0908 2024-11-16T09:59:23,796 INFO [StoreOpener-ffe1934c280093c181c2b062a05e0908-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region ffe1934c280093c181c2b062a05e0908 columnFamilyName C 2024-11-16T09:59:23,797 DEBUG [StoreOpener-ffe1934c280093c181c2b062a05e0908-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T09:59:23,797 INFO [StoreOpener-ffe1934c280093c181c2b062a05e0908-1 {}] regionserver.HStore(327): Store=ffe1934c280093c181c2b062a05e0908/C, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-16T09:59:23,797 DEBUG [RS_OPEN_REGION-regionserver/037c58a19ddb:0-0 {event_type=M_RS_OPEN_REGION, pid=91}] regionserver.HRegion(1038): replaying wal for ffe1934c280093c181c2b062a05e0908 2024-11-16T09:59:23,797 DEBUG [RS_OPEN_REGION-regionserver/037c58a19ddb:0-0 {event_type=M_RS_OPEN_REGION, pid=91}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/ffe1934c280093c181c2b062a05e0908 2024-11-16T09:59:23,798 DEBUG [RS_OPEN_REGION-regionserver/037c58a19ddb:0-0 {event_type=M_RS_OPEN_REGION, pid=91}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/ffe1934c280093c181c2b062a05e0908 2024-11-16T09:59:23,798 DEBUG [RS_OPEN_REGION-regionserver/037c58a19ddb:0-0 {event_type=M_RS_OPEN_REGION, pid=91}] regionserver.HRegion(1048): stopping wal replay for ffe1934c280093c181c2b062a05e0908 2024-11-16T09:59:23,798 DEBUG [RS_OPEN_REGION-regionserver/037c58a19ddb:0-0 {event_type=M_RS_OPEN_REGION, pid=91}] regionserver.HRegion(1060): Cleaning up temporary data for ffe1934c280093c181c2b062a05e0908 2024-11-16T09:59:23,799 DEBUG [RS_OPEN_REGION-regionserver/037c58a19ddb:0-0 {event_type=M_RS_OPEN_REGION, pid=91}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table TestAcidGuarantees descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-11-16T09:59:23,799 DEBUG [RS_OPEN_REGION-regionserver/037c58a19ddb:0-0 {event_type=M_RS_OPEN_REGION, pid=91}] regionserver.HRegion(1093): writing seq id for ffe1934c280093c181c2b062a05e0908 2024-11-16T09:59:23,801 DEBUG [RS_OPEN_REGION-regionserver/037c58a19ddb:0-0 {event_type=M_RS_OPEN_REGION, pid=91}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/ffe1934c280093c181c2b062a05e0908/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-16T09:59:23,801 INFO [RS_OPEN_REGION-regionserver/037c58a19ddb:0-0 {event_type=M_RS_OPEN_REGION, pid=91}] regionserver.HRegion(1114): Opened ffe1934c280093c181c2b062a05e0908; next sequenceid=2; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=69262401, jitterRate=0.0320902019739151}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-11-16T09:59:23,801 DEBUG [RS_OPEN_REGION-regionserver/037c58a19ddb:0-0 {event_type=M_RS_OPEN_REGION, pid=91}] regionserver.HRegion(1122): Running coprocessor post-open hooks for ffe1934c280093c181c2b062a05e0908 2024-11-16T09:59:23,802 DEBUG [RS_OPEN_REGION-regionserver/037c58a19ddb:0-0 {event_type=M_RS_OPEN_REGION, pid=91}] regionserver.HRegion(1006): Region open journal for ffe1934c280093c181c2b062a05e0908: Running coprocessor pre-open hook at 1731751163792Writing region info on filesystem at 1731751163792Initializing all the Stores at 1731751163792Instantiating store for column family {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731751163792Instantiating store for column family {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731751163793 (+1 ms)Instantiating store for column family {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731751163793Cleaning up temporary data from old regions at 1731751163798 (+5 ms)Running coprocessor post-open hooks at 1731751163801 (+3 ms)Region opened successfully at 1731751163802 (+1 ms) 2024-11-16T09:59:23,803 INFO [RS_OPEN_REGION-regionserver/037c58a19ddb:0-0 {event_type=M_RS_OPEN_REGION, pid=91}] regionserver.HRegionServer(2236): Post open deploy tasks for TestAcidGuarantees,,1731751162662.ffe1934c280093c181c2b062a05e0908., pid=91, masterSystemTime=1731751163788 2024-11-16T09:59:23,803 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37877 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=89 2024-11-16T09:59:23,804 DEBUG [RS_OPEN_REGION-regionserver/037c58a19ddb:0-0 {event_type=M_RS_OPEN_REGION, pid=91}] regionserver.HRegionServer(2266): Finished post open deploy task for TestAcidGuarantees,,1731751162662.ffe1934c280093c181c2b062a05e0908. 2024-11-16T09:59:23,804 INFO [RS_OPEN_REGION-regionserver/037c58a19ddb:0-0 {event_type=M_RS_OPEN_REGION, pid=91}] handler.AssignRegionHandler(153): Opened TestAcidGuarantees,,1731751162662.ffe1934c280093c181c2b062a05e0908. 2024-11-16T09:59:23,805 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=90 updating hbase:meta row=ffe1934c280093c181c2b062a05e0908, regionState=OPEN, openSeqNum=2, regionLocation=037c58a19ddb,37267,1731751077844 2024-11-16T09:59:23,806 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=91, ppid=90, state=RUNNABLE, hasLock=false; OpenRegionProcedure ffe1934c280093c181c2b062a05e0908, server=037c58a19ddb,37267,1731751077844 because future has completed 2024-11-16T09:59:23,808 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=91, resume processing ppid=90 2024-11-16T09:59:23,808 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=91, ppid=90, state=SUCCESS, hasLock=false; OpenRegionProcedure ffe1934c280093c181c2b062a05e0908, server=037c58a19ddb,37267,1731751077844 in 170 msec 2024-11-16T09:59:23,810 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=90, resume processing ppid=89 2024-11-16T09:59:23,810 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=90, ppid=89, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=ffe1934c280093c181c2b062a05e0908, ASSIGN in 326 msec 2024-11-16T09:59:23,810 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=89, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-11-16T09:59:23,811 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731751163810"}]},"ts":"1731751163810"} 2024-11-16T09:59:23,812 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLED in hbase:meta 2024-11-16T09:59:23,812 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=89, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_POST_OPERATION 2024-11-16T09:59:23,813 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=89, state=SUCCESS, hasLock=false; CreateTableProcedure table=TestAcidGuarantees in 1.1500 sec 2024-11-16T09:59:24,812 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37877 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=89 2024-11-16T09:59:24,813 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: default:TestAcidGuarantees completed 2024-11-16T09:59:24,814 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5d3de22, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T09:59:24,814 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 037c58a19ddb,37877,-1 for getting cluster id 2024-11-16T09:59:24,814 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-16T09:59:24,815 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '1fe46ee8-cb44-44b1-8251-37103629f786' 2024-11-16T09:59:24,815 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-16T09:59:24,815 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "1fe46ee8-cb44-44b1-8251-37103629f786" 2024-11-16T09:59:24,815 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1041510, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T09:59:24,815 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [037c58a19ddb,37877,-1] 2024-11-16T09:59:24,816 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-16T09:59:24,816 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T09:59:24,817 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:38704, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-16T09:59:24,817 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@51947ebd, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T09:59:24,817 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-16T09:59:24,818 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=037c58a19ddb,37267,1731751077844, seqNum=-1] 2024-11-16T09:59:24,818 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-16T09:59:24,819 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:43310, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-16T09:59:24,820 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-11-16T09:59:24,820 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncConnectionImpl(321): The fetched master address is 037c58a19ddb,37877,1731751077131 2024-11-16T09:59:24,820 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@6b123d53 2024-11-16T09:59:24,821 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-11-16T09:59:24,821 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:38720, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-11-16T09:59:24,823 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37877 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (131072) is too small, which might cause very frequent flushing. 2024-11-16T09:59:24,823 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37877 {}] master.HMaster$14(2863): Client=jenkins//172.17.0.2 modify table TestAcidGuarantees from 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'NONE', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} to 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'NONE', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', MOB_THRESHOLD => '4', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', IS_MOB => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-16T09:59:24,824 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37877 {}] procedure2.ProcedureExecutor(1139): Stored pid=92, state=RUNNABLE:MODIFY_TABLE_PREPARE, hasLock=false; ModifyTableProcedure table=TestAcidGuarantees 2024-11-16T09:59:24,832 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37591 is added to blk_1073742170_1346 (size=995) 2024-11-16T09:59:25,234 DEBUG [PEWorker-5 {}] util.FSTableDescriptors(591): Deleted hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/.tabledesc/.tableinfo.0000000001.959 2024-11-16T09:59:25,234 INFO [PEWorker-5 {}] util.FSTableDescriptors(361): Updated tableinfo=hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/.tabledesc/.tableinfo.0000000002.995 2024-11-16T09:59:25,236 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=93, ppid=92, state=RUNNABLE:REOPEN_TABLE_REGIONS_GET_REGIONS, hasLock=false; ReopenTableRegionsProcedure table=TestAcidGuarantees}] 2024-11-16T09:59:25,239 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=94, ppid=93, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=ffe1934c280093c181c2b062a05e0908, REOPEN/MOVE}] 2024-11-16T09:59:25,239 INFO [PEWorker-3 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=94, ppid=93, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=ffe1934c280093c181c2b062a05e0908, REOPEN/MOVE 2024-11-16T09:59:25,240 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=94 updating hbase:meta row=ffe1934c280093c181c2b062a05e0908, regionState=CLOSING, regionLocation=037c58a19ddb,37267,1731751077844 2024-11-16T09:59:25,241 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=94, ppid=93, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=ffe1934c280093c181c2b062a05e0908, REOPEN/MOVE because future has completed 2024-11-16T09:59:25,241 DEBUG [PEWorker-2 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-11-16T09:59:25,242 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=95, ppid=94, state=RUNNABLE, hasLock=false; CloseRegionProcedure ffe1934c280093c181c2b062a05e0908, server=037c58a19ddb,37267,1731751077844}] 2024-11-16T09:59:25,394 INFO [RS_CLOSE_REGION-regionserver/037c58a19ddb:0-0 {event_type=M_RS_CLOSE_REGION, pid=95}] handler.UnassignRegionHandler(122): Close ffe1934c280093c181c2b062a05e0908 2024-11-16T09:59:25,394 DEBUG [RS_CLOSE_REGION-regionserver/037c58a19ddb:0-0 {event_type=M_RS_CLOSE_REGION, pid=95}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-11-16T09:59:25,394 DEBUG [RS_CLOSE_REGION-regionserver/037c58a19ddb:0-0 {event_type=M_RS_CLOSE_REGION, pid=95}] regionserver.HRegion(1722): Closing ffe1934c280093c181c2b062a05e0908, disabling compactions & flushes 2024-11-16T09:59:25,394 INFO [RS_CLOSE_REGION-regionserver/037c58a19ddb:0-0 {event_type=M_RS_CLOSE_REGION, pid=95}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1731751162662.ffe1934c280093c181c2b062a05e0908. 2024-11-16T09:59:25,394 DEBUG [RS_CLOSE_REGION-regionserver/037c58a19ddb:0-0 {event_type=M_RS_CLOSE_REGION, pid=95}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1731751162662.ffe1934c280093c181c2b062a05e0908. 2024-11-16T09:59:25,394 DEBUG [RS_CLOSE_REGION-regionserver/037c58a19ddb:0-0 {event_type=M_RS_CLOSE_REGION, pid=95}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1731751162662.ffe1934c280093c181c2b062a05e0908. after waiting 0 ms 2024-11-16T09:59:25,394 DEBUG [RS_CLOSE_REGION-regionserver/037c58a19ddb:0-0 {event_type=M_RS_CLOSE_REGION, pid=95}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1731751162662.ffe1934c280093c181c2b062a05e0908. 2024-11-16T09:59:25,397 DEBUG [RS_CLOSE_REGION-regionserver/037c58a19ddb:0-0 {event_type=M_RS_CLOSE_REGION, pid=95}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/ffe1934c280093c181c2b062a05e0908/recovered.edits/4.seqid, newMaxSeqId=4, maxSeqId=1 2024-11-16T09:59:25,398 INFO [RS_CLOSE_REGION-regionserver/037c58a19ddb:0-0 {event_type=M_RS_CLOSE_REGION, pid=95}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1731751162662.ffe1934c280093c181c2b062a05e0908. 2024-11-16T09:59:25,398 DEBUG [RS_CLOSE_REGION-regionserver/037c58a19ddb:0-0 {event_type=M_RS_CLOSE_REGION, pid=95}] regionserver.HRegion(1676): Region close journal for ffe1934c280093c181c2b062a05e0908: Waiting for close lock at 1731751165394Running coprocessor pre-close hooks at 1731751165394Disabling compacts and flushes for region at 1731751165394Disabling writes for close at 1731751165394Writing region close event to WAL at 1731751165394Running coprocessor post-close hooks at 1731751165398 (+4 ms)Closed at 1731751165398 2024-11-16T09:59:25,398 WARN [RS_CLOSE_REGION-regionserver/037c58a19ddb:0-0 {event_type=M_RS_CLOSE_REGION, pid=95}] regionserver.HRegionServer(3299): Not adding moved region record: ffe1934c280093c181c2b062a05e0908 to self. 2024-11-16T09:59:25,400 INFO [RS_CLOSE_REGION-regionserver/037c58a19ddb:0-0 {event_type=M_RS_CLOSE_REGION, pid=95}] handler.UnassignRegionHandler(157): Closed ffe1934c280093c181c2b062a05e0908 2024-11-16T09:59:25,400 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=94 updating hbase:meta row=ffe1934c280093c181c2b062a05e0908, regionState=CLOSED 2024-11-16T09:59:25,401 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=95, ppid=94, state=RUNNABLE, hasLock=false; CloseRegionProcedure ffe1934c280093c181c2b062a05e0908, server=037c58a19ddb,37267,1731751077844 because future has completed 2024-11-16T09:59:25,404 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=95, resume processing ppid=94 2024-11-16T09:59:25,404 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=95, ppid=94, state=SUCCESS, hasLock=false; CloseRegionProcedure ffe1934c280093c181c2b062a05e0908, server=037c58a19ddb,37267,1731751077844 in 161 msec 2024-11-16T09:59:25,404 INFO [PEWorker-3 {}] assignment.TransitRegionStateProcedure(269): Starting pid=94, ppid=93, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=ffe1934c280093c181c2b062a05e0908, REOPEN/MOVE; state=CLOSED, location=037c58a19ddb,37267,1731751077844; forceNewPlan=false, retain=true 2024-11-16T09:59:25,555 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=94 updating hbase:meta row=ffe1934c280093c181c2b062a05e0908, regionState=OPENING, regionLocation=037c58a19ddb,37267,1731751077844 2024-11-16T09:59:25,557 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=94, ppid=93, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=ffe1934c280093c181c2b062a05e0908, REOPEN/MOVE because future has completed 2024-11-16T09:59:25,557 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=96, ppid=94, state=RUNNABLE, hasLock=false; OpenRegionProcedure ffe1934c280093c181c2b062a05e0908, server=037c58a19ddb,37267,1731751077844}] 2024-11-16T09:59:25,711 INFO [RS_OPEN_REGION-regionserver/037c58a19ddb:0-0 {event_type=M_RS_OPEN_REGION, pid=96}] handler.AssignRegionHandler(132): Open TestAcidGuarantees,,1731751162662.ffe1934c280093c181c2b062a05e0908. 2024-11-16T09:59:25,711 DEBUG [RS_OPEN_REGION-regionserver/037c58a19ddb:0-0 {event_type=M_RS_OPEN_REGION, pid=96}] regionserver.HRegion(7752): Opening region: {ENCODED => ffe1934c280093c181c2b062a05e0908, NAME => 'TestAcidGuarantees,,1731751162662.ffe1934c280093c181c2b062a05e0908.', STARTKEY => '', ENDKEY => ''} 2024-11-16T09:59:25,712 DEBUG [RS_OPEN_REGION-regionserver/037c58a19ddb:0-0 {event_type=M_RS_OPEN_REGION, pid=96}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestAcidGuarantees ffe1934c280093c181c2b062a05e0908 2024-11-16T09:59:25,712 DEBUG [RS_OPEN_REGION-regionserver/037c58a19ddb:0-0 {event_type=M_RS_OPEN_REGION, pid=96}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1731751162662.ffe1934c280093c181c2b062a05e0908.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-16T09:59:25,712 DEBUG [RS_OPEN_REGION-regionserver/037c58a19ddb:0-0 {event_type=M_RS_OPEN_REGION, pid=96}] regionserver.HRegion(7794): checking encryption for ffe1934c280093c181c2b062a05e0908 2024-11-16T09:59:25,712 DEBUG [RS_OPEN_REGION-regionserver/037c58a19ddb:0-0 {event_type=M_RS_OPEN_REGION, pid=96}] regionserver.HRegion(7797): checking classloading for ffe1934c280093c181c2b062a05e0908 2024-11-16T09:59:25,713 INFO [StoreOpener-ffe1934c280093c181c2b062a05e0908-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family A of region ffe1934c280093c181c2b062a05e0908 2024-11-16T09:59:25,714 INFO [StoreOpener-ffe1934c280093c181c2b062a05e0908-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region ffe1934c280093c181c2b062a05e0908 columnFamilyName A 2024-11-16T09:59:25,714 DEBUG [StoreOpener-ffe1934c280093c181c2b062a05e0908-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T09:59:25,715 INFO [StoreOpener-ffe1934c280093c181c2b062a05e0908-1 {}] regionserver.HStore(327): Store=ffe1934c280093c181c2b062a05e0908/A, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-16T09:59:25,715 INFO [StoreOpener-ffe1934c280093c181c2b062a05e0908-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family B of region ffe1934c280093c181c2b062a05e0908 2024-11-16T09:59:25,716 INFO [StoreOpener-ffe1934c280093c181c2b062a05e0908-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region ffe1934c280093c181c2b062a05e0908 columnFamilyName B 2024-11-16T09:59:25,716 DEBUG [StoreOpener-ffe1934c280093c181c2b062a05e0908-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T09:59:25,716 INFO [StoreOpener-ffe1934c280093c181c2b062a05e0908-1 {}] regionserver.HStore(327): Store=ffe1934c280093c181c2b062a05e0908/B, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-16T09:59:25,716 INFO [StoreOpener-ffe1934c280093c181c2b062a05e0908-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family C of region ffe1934c280093c181c2b062a05e0908 2024-11-16T09:59:25,717 INFO [StoreOpener-ffe1934c280093c181c2b062a05e0908-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region ffe1934c280093c181c2b062a05e0908 columnFamilyName C 2024-11-16T09:59:25,717 DEBUG [StoreOpener-ffe1934c280093c181c2b062a05e0908-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T09:59:25,717 INFO [StoreOpener-ffe1934c280093c181c2b062a05e0908-1 {}] regionserver.HStore(327): Store=ffe1934c280093c181c2b062a05e0908/C, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-16T09:59:25,717 DEBUG [RS_OPEN_REGION-regionserver/037c58a19ddb:0-0 {event_type=M_RS_OPEN_REGION, pid=96}] regionserver.HRegion(1038): replaying wal for ffe1934c280093c181c2b062a05e0908 2024-11-16T09:59:25,718 DEBUG [RS_OPEN_REGION-regionserver/037c58a19ddb:0-0 {event_type=M_RS_OPEN_REGION, pid=96}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/ffe1934c280093c181c2b062a05e0908 2024-11-16T09:59:25,719 DEBUG [RS_OPEN_REGION-regionserver/037c58a19ddb:0-0 {event_type=M_RS_OPEN_REGION, pid=96}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/ffe1934c280093c181c2b062a05e0908 2024-11-16T09:59:25,719 DEBUG [RS_OPEN_REGION-regionserver/037c58a19ddb:0-0 {event_type=M_RS_OPEN_REGION, pid=96}] regionserver.HRegion(1048): stopping wal replay for ffe1934c280093c181c2b062a05e0908 2024-11-16T09:59:25,719 DEBUG [RS_OPEN_REGION-regionserver/037c58a19ddb:0-0 {event_type=M_RS_OPEN_REGION, pid=96}] regionserver.HRegion(1060): Cleaning up temporary data for ffe1934c280093c181c2b062a05e0908 2024-11-16T09:59:25,720 DEBUG [RS_OPEN_REGION-regionserver/037c58a19ddb:0-0 {event_type=M_RS_OPEN_REGION, pid=96}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table TestAcidGuarantees descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-11-16T09:59:25,721 DEBUG [RS_OPEN_REGION-regionserver/037c58a19ddb:0-0 {event_type=M_RS_OPEN_REGION, pid=96}] regionserver.HRegion(1093): writing seq id for ffe1934c280093c181c2b062a05e0908 2024-11-16T09:59:25,721 INFO [RS_OPEN_REGION-regionserver/037c58a19ddb:0-0 {event_type=M_RS_OPEN_REGION, pid=96}] regionserver.HRegion(1114): Opened ffe1934c280093c181c2b062a05e0908; next sequenceid=5; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=69736553, jitterRate=0.039155617356300354}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-11-16T09:59:25,721 DEBUG [RS_OPEN_REGION-regionserver/037c58a19ddb:0-0 {event_type=M_RS_OPEN_REGION, pid=96}] regionserver.HRegion(1122): Running coprocessor post-open hooks for ffe1934c280093c181c2b062a05e0908 2024-11-16T09:59:25,722 DEBUG [RS_OPEN_REGION-regionserver/037c58a19ddb:0-0 {event_type=M_RS_OPEN_REGION, pid=96}] regionserver.HRegion(1006): Region open journal for ffe1934c280093c181c2b062a05e0908: Running coprocessor pre-open hook at 1731751165712Writing region info on filesystem at 1731751165712Initializing all the Stores at 1731751165713 (+1 ms)Instantiating store for column family {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', MOB_THRESHOLD => '4', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', IS_MOB => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731751165713Instantiating store for column family {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731751165713Instantiating store for column family {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731751165713Cleaning up temporary data from old regions at 1731751165719 (+6 ms)Running coprocessor post-open hooks at 1731751165721 (+2 ms)Region opened successfully at 1731751165722 (+1 ms) 2024-11-16T09:59:25,723 INFO [RS_OPEN_REGION-regionserver/037c58a19ddb:0-0 {event_type=M_RS_OPEN_REGION, pid=96}] regionserver.HRegionServer(2236): Post open deploy tasks for TestAcidGuarantees,,1731751162662.ffe1934c280093c181c2b062a05e0908., pid=96, masterSystemTime=1731751165708 2024-11-16T09:59:25,724 DEBUG [RS_OPEN_REGION-regionserver/037c58a19ddb:0-0 {event_type=M_RS_OPEN_REGION, pid=96}] regionserver.HRegionServer(2266): Finished post open deploy task for TestAcidGuarantees,,1731751162662.ffe1934c280093c181c2b062a05e0908. 2024-11-16T09:59:25,724 INFO [RS_OPEN_REGION-regionserver/037c58a19ddb:0-0 {event_type=M_RS_OPEN_REGION, pid=96}] handler.AssignRegionHandler(153): Opened TestAcidGuarantees,,1731751162662.ffe1934c280093c181c2b062a05e0908. 2024-11-16T09:59:25,725 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=94 updating hbase:meta row=ffe1934c280093c181c2b062a05e0908, regionState=OPEN, openSeqNum=5, regionLocation=037c58a19ddb,37267,1731751077844 2024-11-16T09:59:25,726 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=96, ppid=94, state=RUNNABLE, hasLock=false; OpenRegionProcedure ffe1934c280093c181c2b062a05e0908, server=037c58a19ddb,37267,1731751077844 because future has completed 2024-11-16T09:59:25,728 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=96, resume processing ppid=94 2024-11-16T09:59:25,728 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=96, ppid=94, state=SUCCESS, hasLock=false; OpenRegionProcedure ffe1934c280093c181c2b062a05e0908, server=037c58a19ddb,37267,1731751077844 in 170 msec 2024-11-16T09:59:25,729 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=94, resume processing ppid=93 2024-11-16T09:59:25,730 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=94, ppid=93, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=ffe1934c280093c181c2b062a05e0908, REOPEN/MOVE in 489 msec 2024-11-16T09:59:25,731 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=93, resume processing ppid=92 2024-11-16T09:59:25,731 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=93, ppid=92, state=SUCCESS, hasLock=false; ReopenTableRegionsProcedure table=TestAcidGuarantees in 494 msec 2024-11-16T09:59:25,733 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=92, state=SUCCESS, hasLock=false; ModifyTableProcedure table=TestAcidGuarantees in 909 msec 2024-11-16T09:59:25,734 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37877 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=92 2024-11-16T09:59:25,734 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: MODIFY_COLUMN_FAMILY, Table Name: default:TestAcidGuarantees completed 2024-11-16T09:59:25,735 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6c3d4f84, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T09:59:25,735 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 037c58a19ddb,37877,-1 for getting cluster id 2024-11-16T09:59:25,735 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-16T09:59:25,736 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '1fe46ee8-cb44-44b1-8251-37103629f786' 2024-11-16T09:59:25,736 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-16T09:59:25,736 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "1fe46ee8-cb44-44b1-8251-37103629f786" 2024-11-16T09:59:25,736 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@48cdc8d3, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T09:59:25,736 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [037c58a19ddb,37877,-1] 2024-11-16T09:59:25,737 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-16T09:59:25,737 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T09:59:25,737 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:38742, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-16T09:59:25,738 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@76918d39, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T09:59:25,738 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5ad3f2c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T09:59:25,738 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 037c58a19ddb,37877,-1 for getting cluster id 2024-11-16T09:59:25,738 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-16T09:59:25,739 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '1fe46ee8-cb44-44b1-8251-37103629f786' 2024-11-16T09:59:25,739 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-16T09:59:25,739 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "1fe46ee8-cb44-44b1-8251-37103629f786" 2024-11-16T09:59:25,739 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2fd1ce55, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T09:59:25,739 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [037c58a19ddb,37877,-1] 2024-11-16T09:59:25,740 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-16T09:59:25,740 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T09:59:25,740 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:38758, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-16T09:59:25,741 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@51027da6, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T09:59:25,741 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@422fcba0, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T09:59:25,741 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 037c58a19ddb,37877,-1 for getting cluster id 2024-11-16T09:59:25,741 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-16T09:59:25,742 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '1fe46ee8-cb44-44b1-8251-37103629f786' 2024-11-16T09:59:25,742 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-16T09:59:25,742 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "1fe46ee8-cb44-44b1-8251-37103629f786" 2024-11-16T09:59:25,742 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7954deb, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T09:59:25,742 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [037c58a19ddb,37877,-1] 2024-11-16T09:59:25,743 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-16T09:59:25,743 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T09:59:25,743 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:38778, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-16T09:59:25,744 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@70d0ec92, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T09:59:25,744 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@72e71a40, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T09:59:25,744 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 037c58a19ddb,37877,-1 for getting cluster id 2024-11-16T09:59:25,744 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-16T09:59:25,745 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '1fe46ee8-cb44-44b1-8251-37103629f786' 2024-11-16T09:59:25,745 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-16T09:59:25,745 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "1fe46ee8-cb44-44b1-8251-37103629f786" 2024-11-16T09:59:25,745 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@35eb2d4b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T09:59:25,745 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [037c58a19ddb,37877,-1] 2024-11-16T09:59:25,745 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-16T09:59:25,745 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T09:59:25,746 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:38798, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-16T09:59:25,747 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4f2d48b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T09:59:25,747 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7b9dbe2c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T09:59:25,747 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 037c58a19ddb,37877,-1 for getting cluster id 2024-11-16T09:59:25,747 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-16T09:59:25,748 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '1fe46ee8-cb44-44b1-8251-37103629f786' 2024-11-16T09:59:25,748 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-16T09:59:25,748 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "1fe46ee8-cb44-44b1-8251-37103629f786" 2024-11-16T09:59:25,748 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6cb31310, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T09:59:25,748 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [037c58a19ddb,37877,-1] 2024-11-16T09:59:25,748 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-16T09:59:25,748 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T09:59:25,749 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:38814, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-16T09:59:25,749 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3785871d, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T09:59:25,749 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@10e2b5a3, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T09:59:25,750 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 037c58a19ddb,37877,-1 for getting cluster id 2024-11-16T09:59:25,750 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-16T09:59:25,750 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '1fe46ee8-cb44-44b1-8251-37103629f786' 2024-11-16T09:59:25,750 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-16T09:59:25,750 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "1fe46ee8-cb44-44b1-8251-37103629f786" 2024-11-16T09:59:25,750 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@718409ff, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T09:59:25,750 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [037c58a19ddb,37877,-1] 2024-11-16T09:59:25,751 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-16T09:59:25,751 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T09:59:25,751 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:38842, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-16T09:59:25,752 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@177287b4, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T09:59:25,752 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4d41a701, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T09:59:25,752 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 037c58a19ddb,37877,-1 for getting cluster id 2024-11-16T09:59:25,752 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-16T09:59:25,753 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '1fe46ee8-cb44-44b1-8251-37103629f786' 2024-11-16T09:59:25,753 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-16T09:59:25,753 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "1fe46ee8-cb44-44b1-8251-37103629f786" 2024-11-16T09:59:25,753 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6eb38366, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T09:59:25,753 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [037c58a19ddb,37877,-1] 2024-11-16T09:59:25,753 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-16T09:59:25,753 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T09:59:25,754 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:38858, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-16T09:59:25,754 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7a99fd10, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T09:59:25,755 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4b2f46ef, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T09:59:25,755 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 037c58a19ddb,37877,-1 for getting cluster id 2024-11-16T09:59:25,755 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-16T09:59:25,755 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '1fe46ee8-cb44-44b1-8251-37103629f786' 2024-11-16T09:59:25,755 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-16T09:59:25,756 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "1fe46ee8-cb44-44b1-8251-37103629f786" 2024-11-16T09:59:25,756 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5c0f9ac3, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T09:59:25,756 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [037c58a19ddb,37877,-1] 2024-11-16T09:59:25,756 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-16T09:59:25,756 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T09:59:25,757 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:38884, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-16T09:59:25,757 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1da873f8, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T09:59:25,757 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@180bd544, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T09:59:25,757 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 037c58a19ddb,37877,-1 for getting cluster id 2024-11-16T09:59:25,758 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-16T09:59:25,758 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '1fe46ee8-cb44-44b1-8251-37103629f786' 2024-11-16T09:59:25,758 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-16T09:59:25,758 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "1fe46ee8-cb44-44b1-8251-37103629f786" 2024-11-16T09:59:25,758 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@40eff169, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T09:59:25,759 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [037c58a19ddb,37877,-1] 2024-11-16T09:59:25,759 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-16T09:59:25,759 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T09:59:25,759 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:38894, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-16T09:59:25,760 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2a7df5e6, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T09:59:25,760 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2d553196, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T09:59:25,760 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 037c58a19ddb,37877,-1 for getting cluster id 2024-11-16T09:59:25,760 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-16T09:59:25,761 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '1fe46ee8-cb44-44b1-8251-37103629f786' 2024-11-16T09:59:25,761 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-16T09:59:25,761 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "1fe46ee8-cb44-44b1-8251-37103629f786" 2024-11-16T09:59:25,761 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5ed19426, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T09:59:25,761 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [037c58a19ddb,37877,-1] 2024-11-16T09:59:25,762 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-16T09:59:25,762 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T09:59:25,762 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:38918, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-16T09:59:25,763 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@50b475ba, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T09:59:25,763 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37877 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-16T09:59:25,764 DEBUG [Thread-1587 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-16T09:59:25,764 DEBUG [Thread-1572 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-16T09:59:25,764 DEBUG [Thread-1589 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-16T09:59:25,764 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37877 {}] procedure2.ProcedureExecutor(1139): Stored pid=97, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=97, table=TestAcidGuarantees 2024-11-16T09:59:25,765 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=037c58a19ddb,37267,1731751077844, seqNum=-1] 2024-11-16T09:59:25,765 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-16T09:59:25,765 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=037c58a19ddb,37267,1731751077844, seqNum=-1] 2024-11-16T09:59:25,765 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-16T09:59:25,765 DEBUG [Thread-1583 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-16T09:59:25,765 DEBUG [Thread-1585 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-16T09:59:25,766 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=037c58a19ddb,37267,1731751077844, seqNum=-1] 2024-11-16T09:59:25,766 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-16T09:59:25,766 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:43320, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-16T09:59:25,766 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=037c58a19ddb,37267,1731751077844, seqNum=-1] 2024-11-16T09:59:25,766 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:43332, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-16T09:59:25,766 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37877 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=97 2024-11-16T09:59:25,766 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-16T09:59:25,767 DEBUG [Thread-1591 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-16T09:59:25,767 DEBUG [Thread-1580 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-16T09:59:25,767 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:43338, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-16T09:59:25,767 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=97, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=97, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-16T09:59:25,767 DEBUG [Thread-1574 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-16T09:59:25,767 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=037c58a19ddb,37267,1731751077844, seqNum=-1] 2024-11-16T09:59:25,768 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-16T09:59:25,768 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:43348, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-16T09:59:25,768 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=037c58a19ddb,37267,1731751077844, seqNum=-1] 2024-11-16T09:59:25,768 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=037c58a19ddb,37267,1731751077844, seqNum=-1] 2024-11-16T09:59:25,769 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-16T09:59:25,769 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-16T09:59:25,769 DEBUG [Thread-1576 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-16T09:59:25,769 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=037c58a19ddb,37267,1731751077844, seqNum=-1] 2024-11-16T09:59:25,769 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=97, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=97, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-16T09:59:25,769 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-16T09:59:25,769 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=98, ppid=97, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-16T09:59:25,770 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:43386, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-16T09:59:25,770 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=037c58a19ddb,37267,1731751077844, seqNum=-1] 2024-11-16T09:59:25,770 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:43374, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-16T09:59:25,770 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-16T09:59:25,770 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1731751162662.ffe1934c280093c181c2b062a05e0908., hostname=037c58a19ddb,37267,1731751077844, seqNum=5] 2024-11-16T09:59:25,770 DEBUG [Thread-1578 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-16T09:59:25,770 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:43362, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-16T09:59:25,771 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:43402, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-16T09:59:25,771 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1731751162662.ffe1934c280093c181c2b062a05e0908., hostname=037c58a19ddb,37267,1731751077844, seqNum=5] 2024-11-16T09:59:25,771 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1731751162662.ffe1934c280093c181c2b062a05e0908., hostname=037c58a19ddb,37267,1731751077844, seqNum=5] 2024-11-16T09:59:25,771 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:43416, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-16T09:59:25,772 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=037c58a19ddb,37267,1731751077844, seqNum=-1] 2024-11-16T09:59:25,772 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-16T09:59:25,773 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1731751162662.ffe1934c280093c181c2b062a05e0908., hostname=037c58a19ddb,37267,1731751077844, seqNum=5] 2024-11-16T09:59:25,773 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1731751162662.ffe1934c280093c181c2b062a05e0908., hostname=037c58a19ddb,37267,1731751077844, seqNum=5] 2024-11-16T09:59:25,774 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1731751162662.ffe1934c280093c181c2b062a05e0908., hostname=037c58a19ddb,37267,1731751077844, seqNum=5] 2024-11-16T09:59:25,774 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1731751162662.ffe1934c280093c181c2b062a05e0908., hostname=037c58a19ddb,37267,1731751077844, seqNum=5] 2024-11-16T09:59:25,774 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:43418, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-16T09:59:25,775 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1731751162662.ffe1934c280093c181c2b062a05e0908., hostname=037c58a19ddb,37267,1731751077844, seqNum=5] 2024-11-16T09:59:25,775 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1731751162662.ffe1934c280093c181c2b062a05e0908., hostname=037c58a19ddb,37267,1731751077844, seqNum=5] 2024-11-16T09:59:25,777 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1731751162662.ffe1934c280093c181c2b062a05e0908., hostname=037c58a19ddb,37267,1731751077844, seqNum=5] 2024-11-16T09:59:25,784 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] regionserver.HRegion(8855): Flush requested on ffe1934c280093c181c2b062a05e0908 2024-11-16T09:59:25,784 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-16T09:59:25,784 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing ffe1934c280093c181c2b062a05e0908 3/3 column families, dataSize=53.67 KB heapSize=141.38 KB 2024-11-16T09:59:25,814 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411160435bee7c2b743d3892878423fb632a0_ffe1934c280093c181c2b062a05e0908 is 50, key is test_row_0/A:col10/1731751165778/Put/seqid=0 2024-11-16T09:59:25,818 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37591 is added to blk_1073742171_1347 (size=12154) 2024-11-16T09:59:25,819 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T09:59:25,822 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411160435bee7c2b743d3892878423fb632a0_ffe1934c280093c181c2b062a05e0908 to hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411160435bee7c2b743d3892878423fb632a0_ffe1934c280093c181c2b062a05e0908 2024-11-16T09:59:25,824 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/ffe1934c280093c181c2b062a05e0908/.tmp/A/2d827c9c04a74475be2edfc99ef61e4e, store: [table=TestAcidGuarantees family=A region=ffe1934c280093c181c2b062a05e0908] 2024-11-16T09:59:25,825 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/ffe1934c280093c181c2b062a05e0908/.tmp/A/2d827c9c04a74475be2edfc99ef61e4e is 175, key is test_row_0/A:col10/1731751165778/Put/seqid=0 2024-11-16T09:59:25,833 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffe1934c280093c181c2b062a05e0908, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T09:59:25,833 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] ipc.CallRunner(138): callId: 6 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43418 deadline: 1731751225824, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffe1934c280093c181c2b062a05e0908, server=037c58a19ddb,37267,1731751077844 2024-11-16T09:59:25,833 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffe1934c280093c181c2b062a05e0908, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T09:59:25,834 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731751162662.ffe1934c280093c181c2b062a05e0908., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 , the old value is region=TestAcidGuarantees,,1731751162662.ffe1934c280093c181c2b062a05e0908., hostname=037c58a19ddb,37267,1731751077844, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffe1934c280093c181c2b062a05e0908, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:59:25,834 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731751162662.ffe1934c280093c181c2b062a05e0908., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffe1934c280093c181c2b062a05e0908, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:59:25,834 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43416 deadline: 1731751225827, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffe1934c280093c181c2b062a05e0908, server=037c58a19ddb,37267,1731751077844 2024-11-16T09:59:25,834 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731751162662.ffe1934c280093c181c2b062a05e0908., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 because the exception is null or not the one we care about 2024-11-16T09:59:25,834 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731751162662.ffe1934c280093c181c2b062a05e0908., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 , the old value is region=TestAcidGuarantees,,1731751162662.ffe1934c280093c181c2b062a05e0908., hostname=037c58a19ddb,37267,1731751077844, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffe1934c280093c181c2b062a05e0908, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:59:25,834 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731751162662.ffe1934c280093c181c2b062a05e0908., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffe1934c280093c181c2b062a05e0908, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:59:25,834 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731751162662.ffe1934c280093c181c2b062a05e0908., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 because the exception is null or not the one we care about 2024-11-16T09:59:25,837 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37591 is added to blk_1073742172_1348 (size=30955) 2024-11-16T09:59:25,839 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffe1934c280093c181c2b062a05e0908, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T09:59:25,839 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43362 deadline: 1731751225829, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffe1934c280093c181c2b062a05e0908, server=037c58a19ddb,37267,1731751077844 2024-11-16T09:59:25,839 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731751162662.ffe1934c280093c181c2b062a05e0908., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 , the old value is region=TestAcidGuarantees,,1731751162662.ffe1934c280093c181c2b062a05e0908., hostname=037c58a19ddb,37267,1731751077844, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffe1934c280093c181c2b062a05e0908, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:59:25,839 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731751162662.ffe1934c280093c181c2b062a05e0908., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffe1934c280093c181c2b062a05e0908, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:59:25,839 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731751162662.ffe1934c280093c181c2b062a05e0908., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 because the exception is null or not the one we care about 2024-11-16T09:59:25,840 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffe1934c280093c181c2b062a05e0908, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T09:59:25,840 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43402 deadline: 1731751225829, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffe1934c280093c181c2b062a05e0908, server=037c58a19ddb,37267,1731751077844 2024-11-16T09:59:25,840 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731751162662.ffe1934c280093c181c2b062a05e0908., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 , the old value is region=TestAcidGuarantees,,1731751162662.ffe1934c280093c181c2b062a05e0908., hostname=037c58a19ddb,37267,1731751077844, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffe1934c280093c181c2b062a05e0908, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:59:25,840 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffe1934c280093c181c2b062a05e0908, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T09:59:25,840 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731751162662.ffe1934c280093c181c2b062a05e0908., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffe1934c280093c181c2b062a05e0908, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:59:25,840 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731751162662.ffe1934c280093c181c2b062a05e0908., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 because the exception is null or not the one we care about 2024-11-16T09:59:25,840 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43332 deadline: 1731751225830, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffe1934c280093c181c2b062a05e0908, server=037c58a19ddb,37267,1731751077844 2024-11-16T09:59:25,841 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731751162662.ffe1934c280093c181c2b062a05e0908., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 , the old value is region=TestAcidGuarantees,,1731751162662.ffe1934c280093c181c2b062a05e0908., hostname=037c58a19ddb,37267,1731751077844, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffe1934c280093c181c2b062a05e0908, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:59:25,841 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731751162662.ffe1934c280093c181c2b062a05e0908., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffe1934c280093c181c2b062a05e0908, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:59:25,841 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731751162662.ffe1934c280093c181c2b062a05e0908., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 because the exception is null or not the one we care about 2024-11-16T09:59:25,872 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37877 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=97 2024-11-16T09:59:25,924 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=37267 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=98 2024-11-16T09:59:25,925 DEBUG [RS_FLUSH_OPERATIONS-regionserver/037c58a19ddb:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731751162662.ffe1934c280093c181c2b062a05e0908. 2024-11-16T09:59:25,925 DEBUG [RS_FLUSH_OPERATIONS-regionserver/037c58a19ddb:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731751162662.ffe1934c280093c181c2b062a05e0908. as already flushing 2024-11-16T09:59:25,925 DEBUG [RS_FLUSH_OPERATIONS-regionserver/037c58a19ddb:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731751162662.ffe1934c280093c181c2b062a05e0908. 2024-11-16T09:59:25,925 ERROR [RS_FLUSH_OPERATIONS-regionserver/037c58a19ddb:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] handler.RSProcedureHandler(58): pid=98 java.io.IOException: Unable to complete flush {ENCODED => ffe1934c280093c181c2b062a05e0908, NAME => 'TestAcidGuarantees,,1731751162662.ffe1934c280093c181c2b062a05e0908.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T09:59:25,925 DEBUG [RS_FLUSH_OPERATIONS-regionserver/037c58a19ddb:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=98 java.io.IOException: Unable to complete flush {ENCODED => ffe1934c280093c181c2b062a05e0908, NAME => 'TestAcidGuarantees,,1731751162662.ffe1934c280093c181c2b062a05e0908.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T09:59:25,926 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37877 {}] master.HMaster(4177): Remote procedure failed, pid=98 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ffe1934c280093c181c2b062a05e0908, NAME => 'TestAcidGuarantees,,1731751162662.ffe1934c280093c181c2b062a05e0908.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ffe1934c280093c181c2b062a05e0908, NAME => 'TestAcidGuarantees,,1731751162662.ffe1934c280093c181c2b062a05e0908.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T09:59:25,949 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffe1934c280093c181c2b062a05e0908, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T09:59:25,949 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43418 deadline: 1731751225943, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffe1934c280093c181c2b062a05e0908, server=037c58a19ddb,37267,1731751077844 2024-11-16T09:59:25,949 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffe1934c280093c181c2b062a05e0908, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T09:59:25,949 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731751162662.ffe1934c280093c181c2b062a05e0908., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 , the old value is region=TestAcidGuarantees,,1731751162662.ffe1934c280093c181c2b062a05e0908., hostname=037c58a19ddb,37267,1731751077844, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffe1934c280093c181c2b062a05e0908, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:59:25,949 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43416 deadline: 1731751225943, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffe1934c280093c181c2b062a05e0908, server=037c58a19ddb,37267,1731751077844 2024-11-16T09:59:25,950 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731751162662.ffe1934c280093c181c2b062a05e0908., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffe1934c280093c181c2b062a05e0908, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:59:25,950 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731751162662.ffe1934c280093c181c2b062a05e0908., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 because the exception is null or not the one we care about 2024-11-16T09:59:25,950 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731751162662.ffe1934c280093c181c2b062a05e0908., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 , the old value is region=TestAcidGuarantees,,1731751162662.ffe1934c280093c181c2b062a05e0908., hostname=037c58a19ddb,37267,1731751077844, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffe1934c280093c181c2b062a05e0908, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:59:25,950 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731751162662.ffe1934c280093c181c2b062a05e0908., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffe1934c280093c181c2b062a05e0908, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:59:25,950 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffe1934c280093c181c2b062a05e0908, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T09:59:25,950 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731751162662.ffe1934c280093c181c2b062a05e0908., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 because the exception is null or not the one we care about 2024-11-16T09:59:25,950 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43362 deadline: 1731751225943, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffe1934c280093c181c2b062a05e0908, server=037c58a19ddb,37267,1731751077844 2024-11-16T09:59:25,950 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731751162662.ffe1934c280093c181c2b062a05e0908., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 , the old value is region=TestAcidGuarantees,,1731751162662.ffe1934c280093c181c2b062a05e0908., hostname=037c58a19ddb,37267,1731751077844, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffe1934c280093c181c2b062a05e0908, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:59:25,950 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffe1934c280093c181c2b062a05e0908, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T09:59:25,950 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731751162662.ffe1934c280093c181c2b062a05e0908., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffe1934c280093c181c2b062a05e0908, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:59:25,950 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731751162662.ffe1934c280093c181c2b062a05e0908., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 because the exception is null or not the one we care about 2024-11-16T09:59:25,950 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43402 deadline: 1731751225944, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffe1934c280093c181c2b062a05e0908, server=037c58a19ddb,37267,1731751077844 2024-11-16T09:59:25,950 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731751162662.ffe1934c280093c181c2b062a05e0908., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 , the old value is region=TestAcidGuarantees,,1731751162662.ffe1934c280093c181c2b062a05e0908., hostname=037c58a19ddb,37267,1731751077844, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffe1934c280093c181c2b062a05e0908, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:59:25,950 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731751162662.ffe1934c280093c181c2b062a05e0908., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffe1934c280093c181c2b062a05e0908, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:59:25,950 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffe1934c280093c181c2b062a05e0908, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T09:59:25,950 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731751162662.ffe1934c280093c181c2b062a05e0908., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 because the exception is null or not the one we care about 2024-11-16T09:59:25,951 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43332 deadline: 1731751225944, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffe1934c280093c181c2b062a05e0908, server=037c58a19ddb,37267,1731751077844 2024-11-16T09:59:25,951 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731751162662.ffe1934c280093c181c2b062a05e0908., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 , the old value is region=TestAcidGuarantees,,1731751162662.ffe1934c280093c181c2b062a05e0908., hostname=037c58a19ddb,37267,1731751077844, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffe1934c280093c181c2b062a05e0908, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:59:25,951 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731751162662.ffe1934c280093c181c2b062a05e0908., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffe1934c280093c181c2b062a05e0908, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:59:25,951 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731751162662.ffe1934c280093c181c2b062a05e0908., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 because the exception is null or not the one we care about 2024-11-16T09:59:26,078 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=37267 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=98 2024-11-16T09:59:26,078 DEBUG [RS_FLUSH_OPERATIONS-regionserver/037c58a19ddb:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731751162662.ffe1934c280093c181c2b062a05e0908. 2024-11-16T09:59:26,078 DEBUG [RS_FLUSH_OPERATIONS-regionserver/037c58a19ddb:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731751162662.ffe1934c280093c181c2b062a05e0908. as already flushing 2024-11-16T09:59:26,078 DEBUG [RS_FLUSH_OPERATIONS-regionserver/037c58a19ddb:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731751162662.ffe1934c280093c181c2b062a05e0908. 2024-11-16T09:59:26,078 ERROR [RS_FLUSH_OPERATIONS-regionserver/037c58a19ddb:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] handler.RSProcedureHandler(58): pid=98 java.io.IOException: Unable to complete flush {ENCODED => ffe1934c280093c181c2b062a05e0908, NAME => 'TestAcidGuarantees,,1731751162662.ffe1934c280093c181c2b062a05e0908.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T09:59:26,078 DEBUG [RS_FLUSH_OPERATIONS-regionserver/037c58a19ddb:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=98 java.io.IOException: Unable to complete flush {ENCODED => ffe1934c280093c181c2b062a05e0908, NAME => 'TestAcidGuarantees,,1731751162662.ffe1934c280093c181c2b062a05e0908.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T09:59:26,079 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37877 {}] master.HMaster(4177): Remote procedure failed, pid=98 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ffe1934c280093c181c2b062a05e0908, NAME => 'TestAcidGuarantees,,1731751162662.ffe1934c280093c181c2b062a05e0908.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ffe1934c280093c181c2b062a05e0908, NAME => 'TestAcidGuarantees,,1731751162662.ffe1934c280093c181c2b062a05e0908.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T09:59:26,083 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37877 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=97 2024-11-16T09:59:26,156 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffe1934c280093c181c2b062a05e0908, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T09:59:26,156 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43418 deadline: 1731751226152, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffe1934c280093c181c2b062a05e0908, server=037c58a19ddb,37267,1731751077844 2024-11-16T09:59:26,157 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffe1934c280093c181c2b062a05e0908, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T09:59:26,157 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731751162662.ffe1934c280093c181c2b062a05e0908., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 , the old value is region=TestAcidGuarantees,,1731751162662.ffe1934c280093c181c2b062a05e0908., hostname=037c58a19ddb,37267,1731751077844, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffe1934c280093c181c2b062a05e0908, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:59:26,157 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43416 deadline: 1731751226153, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffe1934c280093c181c2b062a05e0908, server=037c58a19ddb,37267,1731751077844 2024-11-16T09:59:26,157 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731751162662.ffe1934c280093c181c2b062a05e0908., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffe1934c280093c181c2b062a05e0908, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:59:26,157 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731751162662.ffe1934c280093c181c2b062a05e0908., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 because the exception is null or not the one we care about 2024-11-16T09:59:26,157 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffe1934c280093c181c2b062a05e0908, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T09:59:26,157 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731751162662.ffe1934c280093c181c2b062a05e0908., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 , the old value is region=TestAcidGuarantees,,1731751162662.ffe1934c280093c181c2b062a05e0908., hostname=037c58a19ddb,37267,1731751077844, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffe1934c280093c181c2b062a05e0908, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:59:26,157 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731751162662.ffe1934c280093c181c2b062a05e0908., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffe1934c280093c181c2b062a05e0908, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:59:26,157 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43402 deadline: 1731751226153, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffe1934c280093c181c2b062a05e0908, server=037c58a19ddb,37267,1731751077844 2024-11-16T09:59:26,157 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731751162662.ffe1934c280093c181c2b062a05e0908., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 because the exception is null or not the one we care about 2024-11-16T09:59:26,157 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731751162662.ffe1934c280093c181c2b062a05e0908., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 , the old value is region=TestAcidGuarantees,,1731751162662.ffe1934c280093c181c2b062a05e0908., hostname=037c58a19ddb,37267,1731751077844, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffe1934c280093c181c2b062a05e0908, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:59:26,157 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731751162662.ffe1934c280093c181c2b062a05e0908., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffe1934c280093c181c2b062a05e0908, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:59:26,157 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731751162662.ffe1934c280093c181c2b062a05e0908., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 because the exception is null or not the one we care about 2024-11-16T09:59:26,166 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffe1934c280093c181c2b062a05e0908, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T09:59:26,166 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43362 deadline: 1731751226163, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffe1934c280093c181c2b062a05e0908, server=037c58a19ddb,37267,1731751077844 2024-11-16T09:59:26,166 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffe1934c280093c181c2b062a05e0908, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T09:59:26,166 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731751162662.ffe1934c280093c181c2b062a05e0908., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 , the old value is region=TestAcidGuarantees,,1731751162662.ffe1934c280093c181c2b062a05e0908., hostname=037c58a19ddb,37267,1731751077844, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffe1934c280093c181c2b062a05e0908, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:59:26,166 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43332 deadline: 1731751226163, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffe1934c280093c181c2b062a05e0908, server=037c58a19ddb,37267,1731751077844 2024-11-16T09:59:26,166 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731751162662.ffe1934c280093c181c2b062a05e0908., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffe1934c280093c181c2b062a05e0908, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:59:26,166 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731751162662.ffe1934c280093c181c2b062a05e0908., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 because the exception is null or not the one we care about 2024-11-16T09:59:26,167 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731751162662.ffe1934c280093c181c2b062a05e0908., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 , the old value is region=TestAcidGuarantees,,1731751162662.ffe1934c280093c181c2b062a05e0908., hostname=037c58a19ddb,37267,1731751077844, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffe1934c280093c181c2b062a05e0908, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:59:26,167 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731751162662.ffe1934c280093c181c2b062a05e0908., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffe1934c280093c181c2b062a05e0908, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:59:26,167 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731751162662.ffe1934c280093c181c2b062a05e0908., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 because the exception is null or not the one we care about 2024-11-16T09:59:26,231 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=37267 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=98 2024-11-16T09:59:26,231 DEBUG [RS_FLUSH_OPERATIONS-regionserver/037c58a19ddb:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731751162662.ffe1934c280093c181c2b062a05e0908. 2024-11-16T09:59:26,231 DEBUG [RS_FLUSH_OPERATIONS-regionserver/037c58a19ddb:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731751162662.ffe1934c280093c181c2b062a05e0908. as already flushing 2024-11-16T09:59:26,232 DEBUG [RS_FLUSH_OPERATIONS-regionserver/037c58a19ddb:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731751162662.ffe1934c280093c181c2b062a05e0908. 2024-11-16T09:59:26,232 ERROR [RS_FLUSH_OPERATIONS-regionserver/037c58a19ddb:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] handler.RSProcedureHandler(58): pid=98 java.io.IOException: Unable to complete flush {ENCODED => ffe1934c280093c181c2b062a05e0908, NAME => 'TestAcidGuarantees,,1731751162662.ffe1934c280093c181c2b062a05e0908.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T09:59:26,232 DEBUG [RS_FLUSH_OPERATIONS-regionserver/037c58a19ddb:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=98 java.io.IOException: Unable to complete flush {ENCODED => ffe1934c280093c181c2b062a05e0908, NAME => 'TestAcidGuarantees,,1731751162662.ffe1934c280093c181c2b062a05e0908.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T09:59:26,232 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37877 {}] master.HMaster(4177): Remote procedure failed, pid=98 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ffe1934c280093c181c2b062a05e0908, NAME => 'TestAcidGuarantees,,1731751162662.ffe1934c280093c181c2b062a05e0908.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ffe1934c280093c181c2b062a05e0908, NAME => 'TestAcidGuarantees,,1731751162662.ffe1934c280093c181c2b062a05e0908.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T09:59:26,235 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtil$FsDatasetAsyncDiskServiceFixer(597): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details. 2024-11-16T09:59:26,237 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=16, memsize=20.1 K, hasBloomFilter=true, into tmp file hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/ffe1934c280093c181c2b062a05e0908/.tmp/A/2d827c9c04a74475be2edfc99ef61e4e 2024-11-16T09:59:26,260 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/ffe1934c280093c181c2b062a05e0908/.tmp/B/38d966a9b1fa4f7684b985da5492c745 is 50, key is test_row_0/B:col10/1731751165778/Put/seqid=0 2024-11-16T09:59:26,264 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37591 is added to blk_1073742173_1349 (size=12001) 2024-11-16T09:59:26,269 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=16 (bloomFilter=true), to=hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/ffe1934c280093c181c2b062a05e0908/.tmp/B/38d966a9b1fa4f7684b985da5492c745 2024-11-16T09:59:26,294 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/ffe1934c280093c181c2b062a05e0908/.tmp/C/5d8b2f5d76744a4a935a06e9f2b023ea is 50, key is test_row_0/C:col10/1731751165778/Put/seqid=0 2024-11-16T09:59:26,298 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37591 is added to blk_1073742174_1350 (size=12001) 2024-11-16T09:59:26,385 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=37267 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=98 2024-11-16T09:59:26,385 DEBUG [RS_FLUSH_OPERATIONS-regionserver/037c58a19ddb:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731751162662.ffe1934c280093c181c2b062a05e0908. 2024-11-16T09:59:26,385 DEBUG [RS_FLUSH_OPERATIONS-regionserver/037c58a19ddb:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731751162662.ffe1934c280093c181c2b062a05e0908. as already flushing 2024-11-16T09:59:26,385 DEBUG [RS_FLUSH_OPERATIONS-regionserver/037c58a19ddb:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731751162662.ffe1934c280093c181c2b062a05e0908. 2024-11-16T09:59:26,385 ERROR [RS_FLUSH_OPERATIONS-regionserver/037c58a19ddb:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] handler.RSProcedureHandler(58): pid=98 java.io.IOException: Unable to complete flush {ENCODED => ffe1934c280093c181c2b062a05e0908, NAME => 'TestAcidGuarantees,,1731751162662.ffe1934c280093c181c2b062a05e0908.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T09:59:26,385 DEBUG [RS_FLUSH_OPERATIONS-regionserver/037c58a19ddb:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=98 java.io.IOException: Unable to complete flush {ENCODED => ffe1934c280093c181c2b062a05e0908, NAME => 'TestAcidGuarantees,,1731751162662.ffe1934c280093c181c2b062a05e0908.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T09:59:26,386 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37877 {}] master.HMaster(4177): Remote procedure failed, pid=98 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ffe1934c280093c181c2b062a05e0908, NAME => 'TestAcidGuarantees,,1731751162662.ffe1934c280093c181c2b062a05e0908.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ffe1934c280093c181c2b062a05e0908, NAME => 'TestAcidGuarantees,,1731751162662.ffe1934c280093c181c2b062a05e0908.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T09:59:26,392 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37877 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=97 2024-11-16T09:59:26,469 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffe1934c280093c181c2b062a05e0908, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T09:59:26,470 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43418 deadline: 1731751226463, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffe1934c280093c181c2b062a05e0908, server=037c58a19ddb,37267,1731751077844 2024-11-16T09:59:26,470 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffe1934c280093c181c2b062a05e0908, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T09:59:26,470 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43416 deadline: 1731751226463, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffe1934c280093c181c2b062a05e0908, server=037c58a19ddb,37267,1731751077844 2024-11-16T09:59:26,470 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731751162662.ffe1934c280093c181c2b062a05e0908., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 , the old value is region=TestAcidGuarantees,,1731751162662.ffe1934c280093c181c2b062a05e0908., hostname=037c58a19ddb,37267,1731751077844, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffe1934c280093c181c2b062a05e0908, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:59:26,470 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731751162662.ffe1934c280093c181c2b062a05e0908., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffe1934c280093c181c2b062a05e0908, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:59:26,470 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731751162662.ffe1934c280093c181c2b062a05e0908., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 because the exception is null or not the one we care about 2024-11-16T09:59:26,470 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffe1934c280093c181c2b062a05e0908, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T09:59:26,470 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731751162662.ffe1934c280093c181c2b062a05e0908., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 , the old value is region=TestAcidGuarantees,,1731751162662.ffe1934c280093c181c2b062a05e0908., hostname=037c58a19ddb,37267,1731751077844, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffe1934c280093c181c2b062a05e0908, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:59:26,470 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43402 deadline: 1731751226463, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffe1934c280093c181c2b062a05e0908, server=037c58a19ddb,37267,1731751077844 2024-11-16T09:59:26,470 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731751162662.ffe1934c280093c181c2b062a05e0908., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffe1934c280093c181c2b062a05e0908, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:59:26,470 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731751162662.ffe1934c280093c181c2b062a05e0908., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 because the exception is null or not the one we care about 2024-11-16T09:59:26,470 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731751162662.ffe1934c280093c181c2b062a05e0908., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 , the old value is region=TestAcidGuarantees,,1731751162662.ffe1934c280093c181c2b062a05e0908., hostname=037c58a19ddb,37267,1731751077844, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffe1934c280093c181c2b062a05e0908, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:59:26,470 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731751162662.ffe1934c280093c181c2b062a05e0908., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffe1934c280093c181c2b062a05e0908, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:59:26,470 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731751162662.ffe1934c280093c181c2b062a05e0908., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 because the exception is null or not the one we care about 2024-11-16T09:59:26,480 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffe1934c280093c181c2b062a05e0908, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T09:59:26,480 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43362 deadline: 1731751226472, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffe1934c280093c181c2b062a05e0908, server=037c58a19ddb,37267,1731751077844 2024-11-16T09:59:26,480 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffe1934c280093c181c2b062a05e0908, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T09:59:26,480 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731751162662.ffe1934c280093c181c2b062a05e0908., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 , the old value is region=TestAcidGuarantees,,1731751162662.ffe1934c280093c181c2b062a05e0908., hostname=037c58a19ddb,37267,1731751077844, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffe1934c280093c181c2b062a05e0908, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:59:26,480 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43332 deadline: 1731751226472, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffe1934c280093c181c2b062a05e0908, server=037c58a19ddb,37267,1731751077844 2024-11-16T09:59:26,480 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731751162662.ffe1934c280093c181c2b062a05e0908., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffe1934c280093c181c2b062a05e0908, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:59:26,480 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731751162662.ffe1934c280093c181c2b062a05e0908., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 because the exception is null or not the one we care about 2024-11-16T09:59:26,480 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731751162662.ffe1934c280093c181c2b062a05e0908., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 , the old value is region=TestAcidGuarantees,,1731751162662.ffe1934c280093c181c2b062a05e0908., hostname=037c58a19ddb,37267,1731751077844, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffe1934c280093c181c2b062a05e0908, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:59:26,480 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731751162662.ffe1934c280093c181c2b062a05e0908., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffe1934c280093c181c2b062a05e0908, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:59:26,480 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731751162662.ffe1934c280093c181c2b062a05e0908., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 because the exception is null or not the one we care about 2024-11-16T09:59:26,538 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=37267 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=98 2024-11-16T09:59:26,538 DEBUG [RS_FLUSH_OPERATIONS-regionserver/037c58a19ddb:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731751162662.ffe1934c280093c181c2b062a05e0908. 2024-11-16T09:59:26,538 DEBUG [RS_FLUSH_OPERATIONS-regionserver/037c58a19ddb:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731751162662.ffe1934c280093c181c2b062a05e0908. as already flushing 2024-11-16T09:59:26,539 DEBUG [RS_FLUSH_OPERATIONS-regionserver/037c58a19ddb:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731751162662.ffe1934c280093c181c2b062a05e0908. 2024-11-16T09:59:26,539 ERROR [RS_FLUSH_OPERATIONS-regionserver/037c58a19ddb:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] handler.RSProcedureHandler(58): pid=98 java.io.IOException: Unable to complete flush {ENCODED => ffe1934c280093c181c2b062a05e0908, NAME => 'TestAcidGuarantees,,1731751162662.ffe1934c280093c181c2b062a05e0908.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T09:59:26,539 DEBUG [RS_FLUSH_OPERATIONS-regionserver/037c58a19ddb:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=98 java.io.IOException: Unable to complete flush {ENCODED => ffe1934c280093c181c2b062a05e0908, NAME => 'TestAcidGuarantees,,1731751162662.ffe1934c280093c181c2b062a05e0908.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T09:59:26,539 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37877 {}] master.HMaster(4177): Remote procedure failed, pid=98 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ffe1934c280093c181c2b062a05e0908, NAME => 'TestAcidGuarantees,,1731751162662.ffe1934c280093c181c2b062a05e0908.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ffe1934c280093c181c2b062a05e0908, NAME => 'TestAcidGuarantees,,1731751162662.ffe1934c280093c181c2b062a05e0908.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T09:59:26,692 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=37267 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=98 2024-11-16T09:59:26,692 DEBUG [RS_FLUSH_OPERATIONS-regionserver/037c58a19ddb:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731751162662.ffe1934c280093c181c2b062a05e0908. 2024-11-16T09:59:26,692 DEBUG [RS_FLUSH_OPERATIONS-regionserver/037c58a19ddb:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731751162662.ffe1934c280093c181c2b062a05e0908. as already flushing 2024-11-16T09:59:26,692 DEBUG [RS_FLUSH_OPERATIONS-regionserver/037c58a19ddb:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731751162662.ffe1934c280093c181c2b062a05e0908. 2024-11-16T09:59:26,692 ERROR [RS_FLUSH_OPERATIONS-regionserver/037c58a19ddb:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] handler.RSProcedureHandler(58): pid=98 java.io.IOException: Unable to complete flush {ENCODED => ffe1934c280093c181c2b062a05e0908, NAME => 'TestAcidGuarantees,,1731751162662.ffe1934c280093c181c2b062a05e0908.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T09:59:26,692 DEBUG [RS_FLUSH_OPERATIONS-regionserver/037c58a19ddb:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=98 java.io.IOException: Unable to complete flush {ENCODED => ffe1934c280093c181c2b062a05e0908, NAME => 'TestAcidGuarantees,,1731751162662.ffe1934c280093c181c2b062a05e0908.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T09:59:26,693 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37877 {}] master.HMaster(4177): Remote procedure failed, pid=98 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ffe1934c280093c181c2b062a05e0908, NAME => 'TestAcidGuarantees,,1731751162662.ffe1934c280093c181c2b062a05e0908.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ffe1934c280093c181c2b062a05e0908, NAME => 'TestAcidGuarantees,,1731751162662.ffe1934c280093c181c2b062a05e0908.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T09:59:26,699 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=16 (bloomFilter=true), to=hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/ffe1934c280093c181c2b062a05e0908/.tmp/C/5d8b2f5d76744a4a935a06e9f2b023ea 2024-11-16T09:59:26,704 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/ffe1934c280093c181c2b062a05e0908/.tmp/A/2d827c9c04a74475be2edfc99ef61e4e as hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/ffe1934c280093c181c2b062a05e0908/A/2d827c9c04a74475be2edfc99ef61e4e 2024-11-16T09:59:26,708 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/ffe1934c280093c181c2b062a05e0908/A/2d827c9c04a74475be2edfc99ef61e4e, entries=150, sequenceid=16, filesize=30.2 K 2024-11-16T09:59:26,708 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/ffe1934c280093c181c2b062a05e0908/.tmp/B/38d966a9b1fa4f7684b985da5492c745 as hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/ffe1934c280093c181c2b062a05e0908/B/38d966a9b1fa4f7684b985da5492c745 2024-11-16T09:59:26,713 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/ffe1934c280093c181c2b062a05e0908/B/38d966a9b1fa4f7684b985da5492c745, entries=150, sequenceid=16, filesize=11.7 K 2024-11-16T09:59:26,714 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/ffe1934c280093c181c2b062a05e0908/.tmp/C/5d8b2f5d76744a4a935a06e9f2b023ea as hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/ffe1934c280093c181c2b062a05e0908/C/5d8b2f5d76744a4a935a06e9f2b023ea 2024-11-16T09:59:26,719 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/ffe1934c280093c181c2b062a05e0908/C/5d8b2f5d76744a4a935a06e9f2b023ea, entries=150, sequenceid=16, filesize=11.7 K 2024-11-16T09:59:26,719 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~158.91 KB/162720, currentSize=140.89 KB/144270 for ffe1934c280093c181c2b062a05e0908 in 935ms, sequenceid=16, compaction requested=false 2024-11-16T09:59:26,720 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for ffe1934c280093c181c2b062a05e0908: 2024-11-16T09:59:26,845 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=37267 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=98 2024-11-16T09:59:26,845 DEBUG [RS_FLUSH_OPERATIONS-regionserver/037c58a19ddb:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731751162662.ffe1934c280093c181c2b062a05e0908. 2024-11-16T09:59:26,845 INFO [RS_FLUSH_OPERATIONS-regionserver/037c58a19ddb:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HRegion(2902): Flushing ffe1934c280093c181c2b062a05e0908 3/3 column families, dataSize=140.89 KB heapSize=369.89 KB 2024-11-16T09:59:26,854 DEBUG [RS_FLUSH_OPERATIONS-regionserver/037c58a19ddb:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241116d2ccce757c644c5aac4dbb8dce2a1858_ffe1934c280093c181c2b062a05e0908 is 50, key is test_row_0/A:col10/1731751165828/Put/seqid=0 2024-11-16T09:59:26,858 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37591 is added to blk_1073742175_1351 (size=12154) 2024-11-16T09:59:26,903 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37877 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=97 2024-11-16T09:59:26,985 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] regionserver.HRegion(8855): Flush requested on ffe1934c280093c181c2b062a05e0908 2024-11-16T09:59:26,985 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731751162662.ffe1934c280093c181c2b062a05e0908. as already flushing 2024-11-16T09:59:27,002 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffe1934c280093c181c2b062a05e0908, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T09:59:27,002 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43416 deadline: 1731751226994, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffe1934c280093c181c2b062a05e0908, server=037c58a19ddb,37267,1731751077844 2024-11-16T09:59:27,002 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffe1934c280093c181c2b062a05e0908, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T09:59:27,002 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43402 deadline: 1731751226996, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffe1934c280093c181c2b062a05e0908, server=037c58a19ddb,37267,1731751077844 2024-11-16T09:59:27,002 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731751162662.ffe1934c280093c181c2b062a05e0908., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 , the old value is region=TestAcidGuarantees,,1731751162662.ffe1934c280093c181c2b062a05e0908., hostname=037c58a19ddb,37267,1731751077844, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffe1934c280093c181c2b062a05e0908, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:59:27,002 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731751162662.ffe1934c280093c181c2b062a05e0908., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffe1934c280093c181c2b062a05e0908, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:59:27,002 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731751162662.ffe1934c280093c181c2b062a05e0908., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 because the exception is null or not the one we care about 2024-11-16T09:59:27,002 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731751162662.ffe1934c280093c181c2b062a05e0908., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 , the old value is region=TestAcidGuarantees,,1731751162662.ffe1934c280093c181c2b062a05e0908., hostname=037c58a19ddb,37267,1731751077844, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffe1934c280093c181c2b062a05e0908, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:59:27,002 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731751162662.ffe1934c280093c181c2b062a05e0908., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffe1934c280093c181c2b062a05e0908, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:59:27,002 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731751162662.ffe1934c280093c181c2b062a05e0908., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 because the exception is null or not the one we care about 2024-11-16T09:59:27,005 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffe1934c280093c181c2b062a05e0908, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T09:59:27,005 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43418 deadline: 1731751226997, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffe1934c280093c181c2b062a05e0908, server=037c58a19ddb,37267,1731751077844 2024-11-16T09:59:27,005 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731751162662.ffe1934c280093c181c2b062a05e0908., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 , the old value is region=TestAcidGuarantees,,1731751162662.ffe1934c280093c181c2b062a05e0908., hostname=037c58a19ddb,37267,1731751077844, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffe1934c280093c181c2b062a05e0908, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:59:27,006 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731751162662.ffe1934c280093c181c2b062a05e0908., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffe1934c280093c181c2b062a05e0908, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:59:27,006 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731751162662.ffe1934c280093c181c2b062a05e0908., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 because the exception is null or not the one we care about 2024-11-16T09:59:27,009 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffe1934c280093c181c2b062a05e0908, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T09:59:27,010 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43362 deadline: 1731751226998, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffe1934c280093c181c2b062a05e0908, server=037c58a19ddb,37267,1731751077844 2024-11-16T09:59:27,010 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731751162662.ffe1934c280093c181c2b062a05e0908., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 , the old value is region=TestAcidGuarantees,,1731751162662.ffe1934c280093c181c2b062a05e0908., hostname=037c58a19ddb,37267,1731751077844, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffe1934c280093c181c2b062a05e0908, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:59:27,010 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731751162662.ffe1934c280093c181c2b062a05e0908., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffe1934c280093c181c2b062a05e0908, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:59:27,010 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731751162662.ffe1934c280093c181c2b062a05e0908., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 because the exception is null or not the one we care about 2024-11-16T09:59:27,010 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffe1934c280093c181c2b062a05e0908, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T09:59:27,010 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43332 deadline: 1731751226999, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffe1934c280093c181c2b062a05e0908, server=037c58a19ddb,37267,1731751077844 2024-11-16T09:59:27,010 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731751162662.ffe1934c280093c181c2b062a05e0908., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 , the old value is region=TestAcidGuarantees,,1731751162662.ffe1934c280093c181c2b062a05e0908., hostname=037c58a19ddb,37267,1731751077844, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffe1934c280093c181c2b062a05e0908, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:59:27,010 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731751162662.ffe1934c280093c181c2b062a05e0908., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffe1934c280093c181c2b062a05e0908, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:59:27,010 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731751162662.ffe1934c280093c181c2b062a05e0908., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 because the exception is null or not the one we care about 2024-11-16T09:59:27,116 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffe1934c280093c181c2b062a05e0908, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T09:59:27,117 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43416 deadline: 1731751227113, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffe1934c280093c181c2b062a05e0908, server=037c58a19ddb,37267,1731751077844 2024-11-16T09:59:27,117 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731751162662.ffe1934c280093c181c2b062a05e0908., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 , the old value is region=TestAcidGuarantees,,1731751162662.ffe1934c280093c181c2b062a05e0908., hostname=037c58a19ddb,37267,1731751077844, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffe1934c280093c181c2b062a05e0908, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:59:27,117 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffe1934c280093c181c2b062a05e0908, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T09:59:27,117 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731751162662.ffe1934c280093c181c2b062a05e0908., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffe1934c280093c181c2b062a05e0908, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:59:27,117 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731751162662.ffe1934c280093c181c2b062a05e0908., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 because the exception is null or not the one we care about 2024-11-16T09:59:27,117 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43402 deadline: 1731751227113, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffe1934c280093c181c2b062a05e0908, server=037c58a19ddb,37267,1731751077844 2024-11-16T09:59:27,117 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffe1934c280093c181c2b062a05e0908, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T09:59:27,117 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43362 deadline: 1731751227113, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffe1934c280093c181c2b062a05e0908, server=037c58a19ddb,37267,1731751077844 2024-11-16T09:59:27,117 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731751162662.ffe1934c280093c181c2b062a05e0908., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 , the old value is region=TestAcidGuarantees,,1731751162662.ffe1934c280093c181c2b062a05e0908., hostname=037c58a19ddb,37267,1731751077844, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffe1934c280093c181c2b062a05e0908, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:59:27,117 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731751162662.ffe1934c280093c181c2b062a05e0908., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffe1934c280093c181c2b062a05e0908, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:59:27,117 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731751162662.ffe1934c280093c181c2b062a05e0908., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 because the exception is null or not the one we care about 2024-11-16T09:59:27,118 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731751162662.ffe1934c280093c181c2b062a05e0908., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 , the old value is region=TestAcidGuarantees,,1731751162662.ffe1934c280093c181c2b062a05e0908., hostname=037c58a19ddb,37267,1731751077844, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffe1934c280093c181c2b062a05e0908, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:59:27,118 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffe1934c280093c181c2b062a05e0908, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T09:59:27,118 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731751162662.ffe1934c280093c181c2b062a05e0908., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffe1934c280093c181c2b062a05e0908, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:59:27,118 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731751162662.ffe1934c280093c181c2b062a05e0908., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 because the exception is null or not the one we care about 2024-11-16T09:59:27,118 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43418 deadline: 1731751227113, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffe1934c280093c181c2b062a05e0908, server=037c58a19ddb,37267,1731751077844 2024-11-16T09:59:27,118 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffe1934c280093c181c2b062a05e0908, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T09:59:27,118 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43332 deadline: 1731751227113, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffe1934c280093c181c2b062a05e0908, server=037c58a19ddb,37267,1731751077844 2024-11-16T09:59:27,118 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731751162662.ffe1934c280093c181c2b062a05e0908., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 , the old value is region=TestAcidGuarantees,,1731751162662.ffe1934c280093c181c2b062a05e0908., hostname=037c58a19ddb,37267,1731751077844, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffe1934c280093c181c2b062a05e0908, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:59:27,118 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731751162662.ffe1934c280093c181c2b062a05e0908., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffe1934c280093c181c2b062a05e0908, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:59:27,118 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731751162662.ffe1934c280093c181c2b062a05e0908., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 because the exception is null or not the one we care about 2024-11-16T09:59:27,118 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731751162662.ffe1934c280093c181c2b062a05e0908., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 , the old value is region=TestAcidGuarantees,,1731751162662.ffe1934c280093c181c2b062a05e0908., hostname=037c58a19ddb,37267,1731751077844, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffe1934c280093c181c2b062a05e0908, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:59:27,118 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731751162662.ffe1934c280093c181c2b062a05e0908., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffe1934c280093c181c2b062a05e0908, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:59:27,118 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731751162662.ffe1934c280093c181c2b062a05e0908., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 because the exception is null or not the one we care about 2024-11-16T09:59:27,259 DEBUG [RS_FLUSH_OPERATIONS-regionserver/037c58a19ddb:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T09:59:27,263 INFO [RS_FLUSH_OPERATIONS-regionserver/037c58a19ddb:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241116d2ccce757c644c5aac4dbb8dce2a1858_ffe1934c280093c181c2b062a05e0908 to hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241116d2ccce757c644c5aac4dbb8dce2a1858_ffe1934c280093c181c2b062a05e0908 2024-11-16T09:59:27,263 DEBUG [RS_FLUSH_OPERATIONS-regionserver/037c58a19ddb:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/ffe1934c280093c181c2b062a05e0908/.tmp/A/32f240e1f90d4f1c826f4682f47a04f0, store: [table=TestAcidGuarantees family=A region=ffe1934c280093c181c2b062a05e0908] 2024-11-16T09:59:27,264 DEBUG [RS_FLUSH_OPERATIONS-regionserver/037c58a19ddb:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/ffe1934c280093c181c2b062a05e0908/.tmp/A/32f240e1f90d4f1c826f4682f47a04f0 is 175, key is test_row_0/A:col10/1731751165828/Put/seqid=0 2024-11-16T09:59:27,268 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37591 is added to blk_1073742176_1352 (size=30955) 2024-11-16T09:59:27,331 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffe1934c280093c181c2b062a05e0908, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T09:59:27,331 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43416 deadline: 1731751227323, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffe1934c280093c181c2b062a05e0908, server=037c58a19ddb,37267,1731751077844 2024-11-16T09:59:27,331 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffe1934c280093c181c2b062a05e0908, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T09:59:27,331 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43402 deadline: 1731751227323, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffe1934c280093c181c2b062a05e0908, server=037c58a19ddb,37267,1731751077844 2024-11-16T09:59:27,331 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731751162662.ffe1934c280093c181c2b062a05e0908., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 , the old value is region=TestAcidGuarantees,,1731751162662.ffe1934c280093c181c2b062a05e0908., hostname=037c58a19ddb,37267,1731751077844, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffe1934c280093c181c2b062a05e0908, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:59:27,331 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731751162662.ffe1934c280093c181c2b062a05e0908., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffe1934c280093c181c2b062a05e0908, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:59:27,331 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731751162662.ffe1934c280093c181c2b062a05e0908., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 because the exception is null or not the one we care about 2024-11-16T09:59:27,331 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffe1934c280093c181c2b062a05e0908, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T09:59:27,331 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731751162662.ffe1934c280093c181c2b062a05e0908., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 , the old value is region=TestAcidGuarantees,,1731751162662.ffe1934c280093c181c2b062a05e0908., hostname=037c58a19ddb,37267,1731751077844, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffe1934c280093c181c2b062a05e0908, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:59:27,331 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43362 deadline: 1731751227323, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffe1934c280093c181c2b062a05e0908, server=037c58a19ddb,37267,1731751077844 2024-11-16T09:59:27,331 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731751162662.ffe1934c280093c181c2b062a05e0908., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffe1934c280093c181c2b062a05e0908, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:59:27,332 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731751162662.ffe1934c280093c181c2b062a05e0908., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 because the exception is null or not the one we care about 2024-11-16T09:59:27,332 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731751162662.ffe1934c280093c181c2b062a05e0908., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 , the old value is region=TestAcidGuarantees,,1731751162662.ffe1934c280093c181c2b062a05e0908., hostname=037c58a19ddb,37267,1731751077844, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffe1934c280093c181c2b062a05e0908, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:59:27,332 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffe1934c280093c181c2b062a05e0908, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T09:59:27,332 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731751162662.ffe1934c280093c181c2b062a05e0908., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffe1934c280093c181c2b062a05e0908, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:59:27,332 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731751162662.ffe1934c280093c181c2b062a05e0908., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 because the exception is null or not the one we care about 2024-11-16T09:59:27,332 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43418 deadline: 1731751227323, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffe1934c280093c181c2b062a05e0908, server=037c58a19ddb,37267,1731751077844 2024-11-16T09:59:27,332 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffe1934c280093c181c2b062a05e0908, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T09:59:27,332 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731751162662.ffe1934c280093c181c2b062a05e0908., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 , the old value is region=TestAcidGuarantees,,1731751162662.ffe1934c280093c181c2b062a05e0908., hostname=037c58a19ddb,37267,1731751077844, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffe1934c280093c181c2b062a05e0908, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:59:27,332 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43332 deadline: 1731751227323, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffe1934c280093c181c2b062a05e0908, server=037c58a19ddb,37267,1731751077844 2024-11-16T09:59:27,332 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731751162662.ffe1934c280093c181c2b062a05e0908., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffe1934c280093c181c2b062a05e0908, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:59:27,332 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731751162662.ffe1934c280093c181c2b062a05e0908., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 because the exception is null or not the one we care about 2024-11-16T09:59:27,332 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731751162662.ffe1934c280093c181c2b062a05e0908., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 , the old value is region=TestAcidGuarantees,,1731751162662.ffe1934c280093c181c2b062a05e0908., hostname=037c58a19ddb,37267,1731751077844, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffe1934c280093c181c2b062a05e0908, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:59:27,332 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731751162662.ffe1934c280093c181c2b062a05e0908., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffe1934c280093c181c2b062a05e0908, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:59:27,332 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731751162662.ffe1934c280093c181c2b062a05e0908., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 because the exception is null or not the one we care about 2024-11-16T09:59:27,648 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffe1934c280093c181c2b062a05e0908, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T09:59:27,648 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43416 deadline: 1731751227643, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffe1934c280093c181c2b062a05e0908, server=037c58a19ddb,37267,1731751077844 2024-11-16T09:59:27,648 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffe1934c280093c181c2b062a05e0908, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T09:59:27,649 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43402 deadline: 1731751227643, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffe1934c280093c181c2b062a05e0908, server=037c58a19ddb,37267,1731751077844 2024-11-16T09:59:27,649 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731751162662.ffe1934c280093c181c2b062a05e0908., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 , the old value is region=TestAcidGuarantees,,1731751162662.ffe1934c280093c181c2b062a05e0908., hostname=037c58a19ddb,37267,1731751077844, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffe1934c280093c181c2b062a05e0908, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:59:27,649 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731751162662.ffe1934c280093c181c2b062a05e0908., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffe1934c280093c181c2b062a05e0908, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:59:27,649 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731751162662.ffe1934c280093c181c2b062a05e0908., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 because the exception is null or not the one we care about 2024-11-16T09:59:27,649 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffe1934c280093c181c2b062a05e0908, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T09:59:27,649 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731751162662.ffe1934c280093c181c2b062a05e0908., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 , the old value is region=TestAcidGuarantees,,1731751162662.ffe1934c280093c181c2b062a05e0908., hostname=037c58a19ddb,37267,1731751077844, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffe1934c280093c181c2b062a05e0908, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:59:27,649 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43362 deadline: 1731751227643, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffe1934c280093c181c2b062a05e0908, server=037c58a19ddb,37267,1731751077844 2024-11-16T09:59:27,649 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731751162662.ffe1934c280093c181c2b062a05e0908., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffe1934c280093c181c2b062a05e0908, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:59:27,649 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731751162662.ffe1934c280093c181c2b062a05e0908., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 because the exception is null or not the one we care about 2024-11-16T09:59:27,649 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffe1934c280093c181c2b062a05e0908, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T09:59:27,649 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731751162662.ffe1934c280093c181c2b062a05e0908., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 , the old value is region=TestAcidGuarantees,,1731751162662.ffe1934c280093c181c2b062a05e0908., hostname=037c58a19ddb,37267,1731751077844, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffe1934c280093c181c2b062a05e0908, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:59:27,649 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43418 deadline: 1731751227643, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffe1934c280093c181c2b062a05e0908, server=037c58a19ddb,37267,1731751077844 2024-11-16T09:59:27,649 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731751162662.ffe1934c280093c181c2b062a05e0908., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffe1934c280093c181c2b062a05e0908, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:59:27,649 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731751162662.ffe1934c280093c181c2b062a05e0908., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 because the exception is null or not the one we care about 2024-11-16T09:59:27,649 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffe1934c280093c181c2b062a05e0908, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T09:59:27,649 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43332 deadline: 1731751227643, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffe1934c280093c181c2b062a05e0908, server=037c58a19ddb,37267,1731751077844 2024-11-16T09:59:27,649 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731751162662.ffe1934c280093c181c2b062a05e0908., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 , the old value is region=TestAcidGuarantees,,1731751162662.ffe1934c280093c181c2b062a05e0908., hostname=037c58a19ddb,37267,1731751077844, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffe1934c280093c181c2b062a05e0908, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:59:27,649 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731751162662.ffe1934c280093c181c2b062a05e0908., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffe1934c280093c181c2b062a05e0908, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:59:27,649 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731751162662.ffe1934c280093c181c2b062a05e0908., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 because the exception is null or not the one we care about 2024-11-16T09:59:27,650 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731751162662.ffe1934c280093c181c2b062a05e0908., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 , the old value is region=TestAcidGuarantees,,1731751162662.ffe1934c280093c181c2b062a05e0908., hostname=037c58a19ddb,37267,1731751077844, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffe1934c280093c181c2b062a05e0908, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:59:27,650 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731751162662.ffe1934c280093c181c2b062a05e0908., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffe1934c280093c181c2b062a05e0908, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:59:27,650 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731751162662.ffe1934c280093c181c2b062a05e0908., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 because the exception is null or not the one we care about 2024-11-16T09:59:27,669 INFO [RS_FLUSH_OPERATIONS-regionserver/037c58a19ddb:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=40, memsize=47.0 K, hasBloomFilter=true, into tmp file hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/ffe1934c280093c181c2b062a05e0908/.tmp/A/32f240e1f90d4f1c826f4682f47a04f0 2024-11-16T09:59:27,676 DEBUG [RS_FLUSH_OPERATIONS-regionserver/037c58a19ddb:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/ffe1934c280093c181c2b062a05e0908/.tmp/B/40ea47dbea964de1a868077b725e0626 is 50, key is test_row_0/B:col10/1731751165828/Put/seqid=0 2024-11-16T09:59:27,686 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37591 is added to blk_1073742177_1353 (size=12001) 2024-11-16T09:59:27,913 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37877 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=97 2024-11-16T09:59:28,006 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-11-16T09:59:28,087 INFO [RS_FLUSH_OPERATIONS-regionserver/037c58a19ddb:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=40 (bloomFilter=true), to=hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/ffe1934c280093c181c2b062a05e0908/.tmp/B/40ea47dbea964de1a868077b725e0626 2024-11-16T09:59:28,095 DEBUG [RS_FLUSH_OPERATIONS-regionserver/037c58a19ddb:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/ffe1934c280093c181c2b062a05e0908/.tmp/C/9ad2541e56924229b4932313705f9787 is 50, key is test_row_0/C:col10/1731751165828/Put/seqid=0 2024-11-16T09:59:28,099 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37591 is added to blk_1073742178_1354 (size=12001) 2024-11-16T09:59:28,160 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffe1934c280093c181c2b062a05e0908, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T09:59:28,161 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43416 deadline: 1731751228153, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffe1934c280093c181c2b062a05e0908, server=037c58a19ddb,37267,1731751077844 2024-11-16T09:59:28,161 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffe1934c280093c181c2b062a05e0908, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T09:59:28,161 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43402 deadline: 1731751228153, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffe1934c280093c181c2b062a05e0908, server=037c58a19ddb,37267,1731751077844 2024-11-16T09:59:28,161 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731751162662.ffe1934c280093c181c2b062a05e0908., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 , the old value is region=TestAcidGuarantees,,1731751162662.ffe1934c280093c181c2b062a05e0908., hostname=037c58a19ddb,37267,1731751077844, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffe1934c280093c181c2b062a05e0908, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:59:28,161 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731751162662.ffe1934c280093c181c2b062a05e0908., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffe1934c280093c181c2b062a05e0908, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:59:28,161 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731751162662.ffe1934c280093c181c2b062a05e0908., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 because the exception is null or not the one we care about 2024-11-16T09:59:28,161 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731751162662.ffe1934c280093c181c2b062a05e0908., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 , the old value is region=TestAcidGuarantees,,1731751162662.ffe1934c280093c181c2b062a05e0908., hostname=037c58a19ddb,37267,1731751077844, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffe1934c280093c181c2b062a05e0908, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:59:28,161 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731751162662.ffe1934c280093c181c2b062a05e0908., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffe1934c280093c181c2b062a05e0908, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:59:28,161 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731751162662.ffe1934c280093c181c2b062a05e0908., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 because the exception is null or not the one we care about 2024-11-16T09:59:28,171 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffe1934c280093c181c2b062a05e0908, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T09:59:28,171 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43362 deadline: 1731751228163, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffe1934c280093c181c2b062a05e0908, server=037c58a19ddb,37267,1731751077844 2024-11-16T09:59:28,171 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731751162662.ffe1934c280093c181c2b062a05e0908., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 , the old value is region=TestAcidGuarantees,,1731751162662.ffe1934c280093c181c2b062a05e0908., hostname=037c58a19ddb,37267,1731751077844, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffe1934c280093c181c2b062a05e0908, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:59:28,171 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffe1934c280093c181c2b062a05e0908, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T09:59:28,172 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731751162662.ffe1934c280093c181c2b062a05e0908., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffe1934c280093c181c2b062a05e0908, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:59:28,172 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731751162662.ffe1934c280093c181c2b062a05e0908., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 because the exception is null or not the one we care about 2024-11-16T09:59:28,171 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffe1934c280093c181c2b062a05e0908, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T09:59:28,172 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43332 deadline: 1731751228163, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffe1934c280093c181c2b062a05e0908, server=037c58a19ddb,37267,1731751077844 2024-11-16T09:59:28,172 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43418 deadline: 1731751228163, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffe1934c280093c181c2b062a05e0908, server=037c58a19ddb,37267,1731751077844 2024-11-16T09:59:28,172 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731751162662.ffe1934c280093c181c2b062a05e0908., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 , the old value is region=TestAcidGuarantees,,1731751162662.ffe1934c280093c181c2b062a05e0908., hostname=037c58a19ddb,37267,1731751077844, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffe1934c280093c181c2b062a05e0908, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:59:28,172 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731751162662.ffe1934c280093c181c2b062a05e0908., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffe1934c280093c181c2b062a05e0908, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:59:28,172 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731751162662.ffe1934c280093c181c2b062a05e0908., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 because the exception is null or not the one we care about 2024-11-16T09:59:28,172 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731751162662.ffe1934c280093c181c2b062a05e0908., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 , the old value is region=TestAcidGuarantees,,1731751162662.ffe1934c280093c181c2b062a05e0908., hostname=037c58a19ddb,37267,1731751077844, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffe1934c280093c181c2b062a05e0908, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:59:28,172 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731751162662.ffe1934c280093c181c2b062a05e0908., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffe1934c280093c181c2b062a05e0908, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:59:28,172 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731751162662.ffe1934c280093c181c2b062a05e0908., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 because the exception is null or not the one we care about 2024-11-16T09:59:28,499 INFO [RS_FLUSH_OPERATIONS-regionserver/037c58a19ddb:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=40 (bloomFilter=true), to=hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/ffe1934c280093c181c2b062a05e0908/.tmp/C/9ad2541e56924229b4932313705f9787 2024-11-16T09:59:28,504 DEBUG [RS_FLUSH_OPERATIONS-regionserver/037c58a19ddb:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/ffe1934c280093c181c2b062a05e0908/.tmp/A/32f240e1f90d4f1c826f4682f47a04f0 as hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/ffe1934c280093c181c2b062a05e0908/A/32f240e1f90d4f1c826f4682f47a04f0 2024-11-16T09:59:28,508 INFO [RS_FLUSH_OPERATIONS-regionserver/037c58a19ddb:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/ffe1934c280093c181c2b062a05e0908/A/32f240e1f90d4f1c826f4682f47a04f0, entries=150, sequenceid=40, filesize=30.2 K 2024-11-16T09:59:28,509 DEBUG [RS_FLUSH_OPERATIONS-regionserver/037c58a19ddb:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/ffe1934c280093c181c2b062a05e0908/.tmp/B/40ea47dbea964de1a868077b725e0626 as hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/ffe1934c280093c181c2b062a05e0908/B/40ea47dbea964de1a868077b725e0626 2024-11-16T09:59:28,512 INFO [RS_FLUSH_OPERATIONS-regionserver/037c58a19ddb:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/ffe1934c280093c181c2b062a05e0908/B/40ea47dbea964de1a868077b725e0626, entries=150, sequenceid=40, filesize=11.7 K 2024-11-16T09:59:28,513 DEBUG [RS_FLUSH_OPERATIONS-regionserver/037c58a19ddb:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/ffe1934c280093c181c2b062a05e0908/.tmp/C/9ad2541e56924229b4932313705f9787 as hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/ffe1934c280093c181c2b062a05e0908/C/9ad2541e56924229b4932313705f9787 2024-11-16T09:59:28,516 INFO [RS_FLUSH_OPERATIONS-regionserver/037c58a19ddb:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/ffe1934c280093c181c2b062a05e0908/C/9ad2541e56924229b4932313705f9787, entries=150, sequenceid=40, filesize=11.7 K 2024-11-16T09:59:28,517 INFO [RS_FLUSH_OPERATIONS-regionserver/037c58a19ddb:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HRegion(3140): Finished flush of dataSize ~140.89 KB/144270, heapSize ~369.84 KB/378720, currentSize=60.38 KB/61830 for ffe1934c280093c181c2b062a05e0908 in 1672ms, sequenceid=40, compaction requested=false 2024-11-16T09:59:28,517 DEBUG [RS_FLUSH_OPERATIONS-regionserver/037c58a19ddb:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HRegion(2603): Flush status journal for ffe1934c280093c181c2b062a05e0908: 2024-11-16T09:59:28,517 DEBUG [RS_FLUSH_OPERATIONS-regionserver/037c58a19ddb:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731751162662.ffe1934c280093c181c2b062a05e0908. 2024-11-16T09:59:28,518 DEBUG [RS_FLUSH_OPERATIONS-regionserver/037c58a19ddb:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=98 2024-11-16T09:59:28,518 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37877 {}] master.HMaster(4169): Remote procedure done, pid=98 2024-11-16T09:59:28,521 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=98, resume processing ppid=97 2024-11-16T09:59:28,521 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=98, ppid=97, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.7500 sec 2024-11-16T09:59:28,523 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=97, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=97, table=TestAcidGuarantees in 2.7580 sec 2024-11-16T09:59:29,178 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] regionserver.HRegion(8855): Flush requested on ffe1934c280093c181c2b062a05e0908 2024-11-16T09:59:29,178 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-16T09:59:29,178 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing ffe1934c280093c181c2b062a05e0908 3/3 column families, dataSize=67.09 KB heapSize=176.53 KB 2024-11-16T09:59:29,187 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411169aa6866b5eee4abdaec697d6914e75ce_ffe1934c280093c181c2b062a05e0908 is 50, key is test_row_0/A:col10/1731751169177/Put/seqid=0 2024-11-16T09:59:29,203 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37591 is added to blk_1073742179_1355 (size=14594) 2024-11-16T09:59:29,204 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T09:59:29,209 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411169aa6866b5eee4abdaec697d6914e75ce_ffe1934c280093c181c2b062a05e0908 to hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411169aa6866b5eee4abdaec697d6914e75ce_ffe1934c280093c181c2b062a05e0908 2024-11-16T09:59:29,210 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/ffe1934c280093c181c2b062a05e0908/.tmp/A/76773910668b4f61841d0bde508173c5, store: [table=TestAcidGuarantees family=A region=ffe1934c280093c181c2b062a05e0908] 2024-11-16T09:59:29,210 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/ffe1934c280093c181c2b062a05e0908/.tmp/A/76773910668b4f61841d0bde508173c5 is 175, key is test_row_0/A:col10/1731751169177/Put/seqid=0 2024-11-16T09:59:29,215 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37591 is added to blk_1073742180_1356 (size=39549) 2024-11-16T09:59:29,215 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=53, memsize=22.4 K, hasBloomFilter=true, into tmp file hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/ffe1934c280093c181c2b062a05e0908/.tmp/A/76773910668b4f61841d0bde508173c5 2024-11-16T09:59:29,223 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/ffe1934c280093c181c2b062a05e0908/.tmp/B/f0e157f5b6574298a053aade1bdb17e8 is 50, key is test_row_0/B:col10/1731751169177/Put/seqid=0 2024-11-16T09:59:29,226 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37591 is added to blk_1073742181_1357 (size=12001) 2024-11-16T09:59:29,256 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffe1934c280093c181c2b062a05e0908, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T09:59:29,256 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43332 deadline: 1731751229241, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffe1934c280093c181c2b062a05e0908, server=037c58a19ddb,37267,1731751077844 2024-11-16T09:59:29,256 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731751162662.ffe1934c280093c181c2b062a05e0908., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 , the old value is region=TestAcidGuarantees,,1731751162662.ffe1934c280093c181c2b062a05e0908., hostname=037c58a19ddb,37267,1731751077844, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffe1934c280093c181c2b062a05e0908, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:59:29,256 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffe1934c280093c181c2b062a05e0908, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T09:59:29,256 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731751162662.ffe1934c280093c181c2b062a05e0908., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffe1934c280093c181c2b062a05e0908, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:59:29,256 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731751162662.ffe1934c280093c181c2b062a05e0908., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 because the exception is null or not the one we care about 2024-11-16T09:59:29,256 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43402 deadline: 1731751229251, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffe1934c280093c181c2b062a05e0908, server=037c58a19ddb,37267,1731751077844 2024-11-16T09:59:29,257 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731751162662.ffe1934c280093c181c2b062a05e0908., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 , the old value is region=TestAcidGuarantees,,1731751162662.ffe1934c280093c181c2b062a05e0908., hostname=037c58a19ddb,37267,1731751077844, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffe1934c280093c181c2b062a05e0908, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:59:29,257 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731751162662.ffe1934c280093c181c2b062a05e0908., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffe1934c280093c181c2b062a05e0908, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:59:29,257 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731751162662.ffe1934c280093c181c2b062a05e0908., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 because the exception is null or not the one we care about 2024-11-16T09:59:29,261 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffe1934c280093c181c2b062a05e0908, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T09:59:29,261 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43416 deadline: 1731751229251, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffe1934c280093c181c2b062a05e0908, server=037c58a19ddb,37267,1731751077844 2024-11-16T09:59:29,261 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffe1934c280093c181c2b062a05e0908, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T09:59:29,261 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43418 deadline: 1731751229255, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffe1934c280093c181c2b062a05e0908, server=037c58a19ddb,37267,1731751077844 2024-11-16T09:59:29,261 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731751162662.ffe1934c280093c181c2b062a05e0908., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 , the old value is region=TestAcidGuarantees,,1731751162662.ffe1934c280093c181c2b062a05e0908., hostname=037c58a19ddb,37267,1731751077844, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffe1934c280093c181c2b062a05e0908, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:59:29,261 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731751162662.ffe1934c280093c181c2b062a05e0908., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffe1934c280093c181c2b062a05e0908, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:59:29,261 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731751162662.ffe1934c280093c181c2b062a05e0908., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 because the exception is null or not the one we care about 2024-11-16T09:59:29,262 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731751162662.ffe1934c280093c181c2b062a05e0908., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 , the old value is region=TestAcidGuarantees,,1731751162662.ffe1934c280093c181c2b062a05e0908., hostname=037c58a19ddb,37267,1731751077844, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffe1934c280093c181c2b062a05e0908, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:59:29,262 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731751162662.ffe1934c280093c181c2b062a05e0908., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffe1934c280093c181c2b062a05e0908, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:59:29,262 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731751162662.ffe1934c280093c181c2b062a05e0908., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 because the exception is null or not the one we care about 2024-11-16T09:59:29,270 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffe1934c280093c181c2b062a05e0908, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T09:59:29,270 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43362 deadline: 1731751229256, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffe1934c280093c181c2b062a05e0908, server=037c58a19ddb,37267,1731751077844 2024-11-16T09:59:29,270 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731751162662.ffe1934c280093c181c2b062a05e0908., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 , the old value is region=TestAcidGuarantees,,1731751162662.ffe1934c280093c181c2b062a05e0908., hostname=037c58a19ddb,37267,1731751077844, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffe1934c280093c181c2b062a05e0908, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:59:29,270 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731751162662.ffe1934c280093c181c2b062a05e0908., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffe1934c280093c181c2b062a05e0908, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:59:29,270 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731751162662.ffe1934c280093c181c2b062a05e0908., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 because the exception is null or not the one we care about 2024-11-16T09:59:29,372 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffe1934c280093c181c2b062a05e0908, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T09:59:29,372 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43332 deadline: 1731751229363, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffe1934c280093c181c2b062a05e0908, server=037c58a19ddb,37267,1731751077844 2024-11-16T09:59:29,372 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffe1934c280093c181c2b062a05e0908, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T09:59:29,372 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43402 deadline: 1731751229363, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffe1934c280093c181c2b062a05e0908, server=037c58a19ddb,37267,1731751077844 2024-11-16T09:59:29,372 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731751162662.ffe1934c280093c181c2b062a05e0908., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 , the old value is region=TestAcidGuarantees,,1731751162662.ffe1934c280093c181c2b062a05e0908., hostname=037c58a19ddb,37267,1731751077844, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffe1934c280093c181c2b062a05e0908, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:59:29,372 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffe1934c280093c181c2b062a05e0908, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T09:59:29,372 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731751162662.ffe1934c280093c181c2b062a05e0908., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffe1934c280093c181c2b062a05e0908, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:59:29,372 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731751162662.ffe1934c280093c181c2b062a05e0908., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 because the exception is null or not the one we care about 2024-11-16T09:59:29,372 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43416 deadline: 1731751229363, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffe1934c280093c181c2b062a05e0908, server=037c58a19ddb,37267,1731751077844 2024-11-16T09:59:29,372 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731751162662.ffe1934c280093c181c2b062a05e0908., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 , the old value is region=TestAcidGuarantees,,1731751162662.ffe1934c280093c181c2b062a05e0908., hostname=037c58a19ddb,37267,1731751077844, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffe1934c280093c181c2b062a05e0908, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:59:29,372 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731751162662.ffe1934c280093c181c2b062a05e0908., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffe1934c280093c181c2b062a05e0908, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:59:29,372 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731751162662.ffe1934c280093c181c2b062a05e0908., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 because the exception is null or not the one we care about 2024-11-16T09:59:29,373 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731751162662.ffe1934c280093c181c2b062a05e0908., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 , the old value is region=TestAcidGuarantees,,1731751162662.ffe1934c280093c181c2b062a05e0908., hostname=037c58a19ddb,37267,1731751077844, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffe1934c280093c181c2b062a05e0908, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:59:29,373 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731751162662.ffe1934c280093c181c2b062a05e0908., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffe1934c280093c181c2b062a05e0908, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:59:29,373 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731751162662.ffe1934c280093c181c2b062a05e0908., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 because the exception is null or not the one we care about 2024-11-16T09:59:29,376 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffe1934c280093c181c2b062a05e0908, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T09:59:29,376 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43418 deadline: 1731751229373, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffe1934c280093c181c2b062a05e0908, server=037c58a19ddb,37267,1731751077844 2024-11-16T09:59:29,376 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffe1934c280093c181c2b062a05e0908, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T09:59:29,377 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731751162662.ffe1934c280093c181c2b062a05e0908., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 , the old value is region=TestAcidGuarantees,,1731751162662.ffe1934c280093c181c2b062a05e0908., hostname=037c58a19ddb,37267,1731751077844, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffe1934c280093c181c2b062a05e0908, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:59:29,377 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731751162662.ffe1934c280093c181c2b062a05e0908., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffe1934c280093c181c2b062a05e0908, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:59:29,377 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43362 deadline: 1731751229373, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffe1934c280093c181c2b062a05e0908, server=037c58a19ddb,37267,1731751077844 2024-11-16T09:59:29,377 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731751162662.ffe1934c280093c181c2b062a05e0908., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 because the exception is null or not the one we care about 2024-11-16T09:59:29,377 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731751162662.ffe1934c280093c181c2b062a05e0908., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 , the old value is region=TestAcidGuarantees,,1731751162662.ffe1934c280093c181c2b062a05e0908., hostname=037c58a19ddb,37267,1731751077844, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffe1934c280093c181c2b062a05e0908, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:59:29,377 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731751162662.ffe1934c280093c181c2b062a05e0908., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffe1934c280093c181c2b062a05e0908, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:59:29,377 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731751162662.ffe1934c280093c181c2b062a05e0908., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 because the exception is null or not the one we care about 2024-11-16T09:59:29,586 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffe1934c280093c181c2b062a05e0908, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T09:59:29,586 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43332 deadline: 1731751229583, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffe1934c280093c181c2b062a05e0908, server=037c58a19ddb,37267,1731751077844 2024-11-16T09:59:29,586 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731751162662.ffe1934c280093c181c2b062a05e0908., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 , the old value is region=TestAcidGuarantees,,1731751162662.ffe1934c280093c181c2b062a05e0908., hostname=037c58a19ddb,37267,1731751077844, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffe1934c280093c181c2b062a05e0908, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:59:29,586 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffe1934c280093c181c2b062a05e0908, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T09:59:29,586 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731751162662.ffe1934c280093c181c2b062a05e0908., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffe1934c280093c181c2b062a05e0908, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:59:29,586 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731751162662.ffe1934c280093c181c2b062a05e0908., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 because the exception is null or not the one we care about 2024-11-16T09:59:29,586 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43402 deadline: 1731751229583, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffe1934c280093c181c2b062a05e0908, server=037c58a19ddb,37267,1731751077844 2024-11-16T09:59:29,587 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffe1934c280093c181c2b062a05e0908, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T09:59:29,587 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731751162662.ffe1934c280093c181c2b062a05e0908., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 , the old value is region=TestAcidGuarantees,,1731751162662.ffe1934c280093c181c2b062a05e0908., hostname=037c58a19ddb,37267,1731751077844, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffe1934c280093c181c2b062a05e0908, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:59:29,587 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731751162662.ffe1934c280093c181c2b062a05e0908., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffe1934c280093c181c2b062a05e0908, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:59:29,587 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731751162662.ffe1934c280093c181c2b062a05e0908., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 because the exception is null or not the one we care about 2024-11-16T09:59:29,587 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43418 deadline: 1731751229583, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffe1934c280093c181c2b062a05e0908, server=037c58a19ddb,37267,1731751077844 2024-11-16T09:59:29,587 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731751162662.ffe1934c280093c181c2b062a05e0908., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 , the old value is region=TestAcidGuarantees,,1731751162662.ffe1934c280093c181c2b062a05e0908., hostname=037c58a19ddb,37267,1731751077844, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffe1934c280093c181c2b062a05e0908, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:59:29,587 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731751162662.ffe1934c280093c181c2b062a05e0908., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffe1934c280093c181c2b062a05e0908, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:59:29,587 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffe1934c280093c181c2b062a05e0908, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T09:59:29,587 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731751162662.ffe1934c280093c181c2b062a05e0908., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 because the exception is null or not the one we care about 2024-11-16T09:59:29,587 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43416 deadline: 1731751229583, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffe1934c280093c181c2b062a05e0908, server=037c58a19ddb,37267,1731751077844 2024-11-16T09:59:29,587 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731751162662.ffe1934c280093c181c2b062a05e0908., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 , the old value is region=TestAcidGuarantees,,1731751162662.ffe1934c280093c181c2b062a05e0908., hostname=037c58a19ddb,37267,1731751077844, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffe1934c280093c181c2b062a05e0908, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:59:29,587 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffe1934c280093c181c2b062a05e0908, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T09:59:29,587 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731751162662.ffe1934c280093c181c2b062a05e0908., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffe1934c280093c181c2b062a05e0908, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:59:29,587 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731751162662.ffe1934c280093c181c2b062a05e0908., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 because the exception is null or not the one we care about 2024-11-16T09:59:29,587 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43362 deadline: 1731751229583, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffe1934c280093c181c2b062a05e0908, server=037c58a19ddb,37267,1731751077844 2024-11-16T09:59:29,588 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731751162662.ffe1934c280093c181c2b062a05e0908., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 , the old value is region=TestAcidGuarantees,,1731751162662.ffe1934c280093c181c2b062a05e0908., hostname=037c58a19ddb,37267,1731751077844, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffe1934c280093c181c2b062a05e0908, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:59:29,588 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731751162662.ffe1934c280093c181c2b062a05e0908., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffe1934c280093c181c2b062a05e0908, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:59:29,588 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731751162662.ffe1934c280093c181c2b062a05e0908., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 because the exception is null or not the one we care about 2024-11-16T09:59:29,626 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=53 (bloomFilter=true), to=hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/ffe1934c280093c181c2b062a05e0908/.tmp/B/f0e157f5b6574298a053aade1bdb17e8 2024-11-16T09:59:29,636 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/ffe1934c280093c181c2b062a05e0908/.tmp/C/7fa927d3441046e0962c03ddbd3858b7 is 50, key is test_row_0/C:col10/1731751169177/Put/seqid=0 2024-11-16T09:59:29,643 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37591 is added to blk_1073742182_1358 (size=12001) 2024-11-16T09:59:29,644 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=53 (bloomFilter=true), to=hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/ffe1934c280093c181c2b062a05e0908/.tmp/C/7fa927d3441046e0962c03ddbd3858b7 2024-11-16T09:59:29,650 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/ffe1934c280093c181c2b062a05e0908/.tmp/A/76773910668b4f61841d0bde508173c5 as hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/ffe1934c280093c181c2b062a05e0908/A/76773910668b4f61841d0bde508173c5 2024-11-16T09:59:29,654 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/ffe1934c280093c181c2b062a05e0908/A/76773910668b4f61841d0bde508173c5, entries=200, sequenceid=53, filesize=38.6 K 2024-11-16T09:59:29,655 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/ffe1934c280093c181c2b062a05e0908/.tmp/B/f0e157f5b6574298a053aade1bdb17e8 as hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/ffe1934c280093c181c2b062a05e0908/B/f0e157f5b6574298a053aade1bdb17e8 2024-11-16T09:59:29,660 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/ffe1934c280093c181c2b062a05e0908/B/f0e157f5b6574298a053aade1bdb17e8, entries=150, sequenceid=53, filesize=11.7 K 2024-11-16T09:59:29,660 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/ffe1934c280093c181c2b062a05e0908/.tmp/C/7fa927d3441046e0962c03ddbd3858b7 as hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/ffe1934c280093c181c2b062a05e0908/C/7fa927d3441046e0962c03ddbd3858b7 2024-11-16T09:59:29,665 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/ffe1934c280093c181c2b062a05e0908/C/7fa927d3441046e0962c03ddbd3858b7, entries=150, sequenceid=53, filesize=11.7 K 2024-11-16T09:59:29,665 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~67.09 KB/68700, heapSize ~176.48 KB/180720, currentSize=134.18 KB/137400 for ffe1934c280093c181c2b062a05e0908 in 487ms, sequenceid=53, compaction requested=true 2024-11-16T09:59:29,666 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for ffe1934c280093c181c2b062a05e0908: 2024-11-16T09:59:29,666 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store ffe1934c280093c181c2b062a05e0908:A, priority=-2147483648, current under compaction store size is 1 2024-11-16T09:59:29,666 DEBUG [RS:0;037c58a19ddb:37267-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-16T09:59:29,666 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-16T09:59:29,666 DEBUG [RS:0;037c58a19ddb:37267-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-16T09:59:29,666 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store ffe1934c280093c181c2b062a05e0908:B, priority=-2147483648, current under compaction store size is 2 2024-11-16T09:59:29,666 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-16T09:59:29,666 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store ffe1934c280093c181c2b062a05e0908:C, priority=-2147483648, current under compaction store size is 3 2024-11-16T09:59:29,666 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-16T09:59:29,667 DEBUG [RS:0;037c58a19ddb:37267-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36003 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-16T09:59:29,667 DEBUG [RS:0;037c58a19ddb:37267-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 101459 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-16T09:59:29,667 DEBUG [RS:0;037c58a19ddb:37267-longCompactions-0 {}] regionserver.HStore(1541): ffe1934c280093c181c2b062a05e0908/B is initiating minor compaction (all files) 2024-11-16T09:59:29,667 DEBUG [RS:0;037c58a19ddb:37267-shortCompactions-0 {}] regionserver.HStore(1541): ffe1934c280093c181c2b062a05e0908/A is initiating minor compaction (all files) 2024-11-16T09:59:29,667 INFO [RS:0;037c58a19ddb:37267-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of ffe1934c280093c181c2b062a05e0908/B in TestAcidGuarantees,,1731751162662.ffe1934c280093c181c2b062a05e0908. 2024-11-16T09:59:29,667 INFO [RS:0;037c58a19ddb:37267-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of ffe1934c280093c181c2b062a05e0908/A in TestAcidGuarantees,,1731751162662.ffe1934c280093c181c2b062a05e0908. 2024-11-16T09:59:29,667 INFO [RS:0;037c58a19ddb:37267-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/ffe1934c280093c181c2b062a05e0908/B/38d966a9b1fa4f7684b985da5492c745, hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/ffe1934c280093c181c2b062a05e0908/B/40ea47dbea964de1a868077b725e0626, hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/ffe1934c280093c181c2b062a05e0908/B/f0e157f5b6574298a053aade1bdb17e8] into tmpdir=hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/ffe1934c280093c181c2b062a05e0908/.tmp, totalSize=35.2 K 2024-11-16T09:59:29,667 INFO [RS:0;037c58a19ddb:37267-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/ffe1934c280093c181c2b062a05e0908/A/2d827c9c04a74475be2edfc99ef61e4e, hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/ffe1934c280093c181c2b062a05e0908/A/32f240e1f90d4f1c826f4682f47a04f0, hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/ffe1934c280093c181c2b062a05e0908/A/76773910668b4f61841d0bde508173c5] into tmpdir=hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/ffe1934c280093c181c2b062a05e0908/.tmp, totalSize=99.1 K 2024-11-16T09:59:29,667 INFO [RS:0;037c58a19ddb:37267-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731751162662.ffe1934c280093c181c2b062a05e0908. 2024-11-16T09:59:29,667 DEBUG [RS:0;037c58a19ddb:37267-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731751162662.ffe1934c280093c181c2b062a05e0908. files: [hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/ffe1934c280093c181c2b062a05e0908/A/2d827c9c04a74475be2edfc99ef61e4e, hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/ffe1934c280093c181c2b062a05e0908/A/32f240e1f90d4f1c826f4682f47a04f0, hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/ffe1934c280093c181c2b062a05e0908/A/76773910668b4f61841d0bde508173c5] 2024-11-16T09:59:29,668 DEBUG [RS:0;037c58a19ddb:37267-longCompactions-0 {}] compactions.Compactor(225): Compacting 38d966a9b1fa4f7684b985da5492c745, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=16, earliestPutTs=1731751165778 2024-11-16T09:59:29,668 DEBUG [RS:0;037c58a19ddb:37267-shortCompactions-0 {}] compactions.Compactor(225): Compacting 2d827c9c04a74475be2edfc99ef61e4e, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=16, earliestPutTs=1731751165778 2024-11-16T09:59:29,668 DEBUG [RS:0;037c58a19ddb:37267-longCompactions-0 {}] compactions.Compactor(225): Compacting 40ea47dbea964de1a868077b725e0626, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=40, earliestPutTs=1731751165822 2024-11-16T09:59:29,668 DEBUG [RS:0;037c58a19ddb:37267-shortCompactions-0 {}] compactions.Compactor(225): Compacting 32f240e1f90d4f1c826f4682f47a04f0, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=40, earliestPutTs=1731751165822 2024-11-16T09:59:29,668 DEBUG [RS:0;037c58a19ddb:37267-longCompactions-0 {}] compactions.Compactor(225): Compacting f0e157f5b6574298a053aade1bdb17e8, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=53, earliestPutTs=1731751166994 2024-11-16T09:59:29,668 DEBUG [RS:0;037c58a19ddb:37267-shortCompactions-0 {}] compactions.Compactor(225): Compacting 76773910668b4f61841d0bde508173c5, keycount=200, bloomtype=ROW, size=38.6 K, encoding=NONE, compression=NONE, seqNum=53, earliestPutTs=1731751166994 2024-11-16T09:59:29,673 INFO [RS:0;037c58a19ddb:37267-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=ffe1934c280093c181c2b062a05e0908] 2024-11-16T09:59:29,676 INFO [RS:0;037c58a19ddb:37267-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): ffe1934c280093c181c2b062a05e0908#B#compaction#309 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-16T09:59:29,676 DEBUG [RS:0;037c58a19ddb:37267-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/ffe1934c280093c181c2b062a05e0908/.tmp/B/be4ceea2e8b74c3fb5af6601e8cc803b is 50, key is test_row_0/B:col10/1731751169177/Put/seqid=0 2024-11-16T09:59:29,678 DEBUG [RS:0;037c58a19ddb:37267-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e202411162061d07152524c11ae7468f8414859fa_ffe1934c280093c181c2b062a05e0908 store=[table=TestAcidGuarantees family=A region=ffe1934c280093c181c2b062a05e0908] 2024-11-16T09:59:29,679 DEBUG [RS:0;037c58a19ddb:37267-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e202411162061d07152524c11ae7468f8414859fa_ffe1934c280093c181c2b062a05e0908, store=[table=TestAcidGuarantees family=A region=ffe1934c280093c181c2b062a05e0908] 2024-11-16T09:59:29,680 DEBUG [RS:0;037c58a19ddb:37267-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411162061d07152524c11ae7468f8414859fa_ffe1934c280093c181c2b062a05e0908 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=ffe1934c280093c181c2b062a05e0908] 2024-11-16T09:59:29,685 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37591 is added to blk_1073742183_1359 (size=12104) 2024-11-16T09:59:29,695 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37591 is added to blk_1073742184_1360 (size=4469) 2024-11-16T09:59:29,696 INFO [RS:0;037c58a19ddb:37267-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): ffe1934c280093c181c2b062a05e0908#A#compaction#308 average throughput is 1.06 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-16T09:59:29,697 DEBUG [RS:0;037c58a19ddb:37267-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/ffe1934c280093c181c2b062a05e0908/.tmp/A/4939aa1e225f4585a4c8405038aea298 is 175, key is test_row_0/A:col10/1731751169177/Put/seqid=0 2024-11-16T09:59:29,700 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37591 is added to blk_1073742185_1361 (size=31058) 2024-11-16T09:59:29,897 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] regionserver.HRegion(8855): Flush requested on ffe1934c280093c181c2b062a05e0908 2024-11-16T09:59:29,897 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-16T09:59:29,897 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing ffe1934c280093c181c2b062a05e0908 3/3 column families, dataSize=140.89 KB heapSize=369.89 KB 2024-11-16T09:59:29,904 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411166857ff57a2f341f699ea264dea73761a_ffe1934c280093c181c2b062a05e0908 is 50, key is test_row_0/A:col10/1731751169896/Put/seqid=0 2024-11-16T09:59:29,908 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37591 is added to blk_1073742186_1362 (size=14594) 2024-11-16T09:59:29,913 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffe1934c280093c181c2b062a05e0908, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T09:59:29,914 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43332 deadline: 1731751229910, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffe1934c280093c181c2b062a05e0908, server=037c58a19ddb,37267,1731751077844 2024-11-16T09:59:29,914 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731751162662.ffe1934c280093c181c2b062a05e0908., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 , the old value is region=TestAcidGuarantees,,1731751162662.ffe1934c280093c181c2b062a05e0908., hostname=037c58a19ddb,37267,1731751077844, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffe1934c280093c181c2b062a05e0908, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:59:29,914 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731751162662.ffe1934c280093c181c2b062a05e0908., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffe1934c280093c181c2b062a05e0908, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:59:29,914 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731751162662.ffe1934c280093c181c2b062a05e0908., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 because the exception is null or not the one we care about 2024-11-16T09:59:29,927 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffe1934c280093c181c2b062a05e0908, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T09:59:29,927 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43362 deadline: 1731751229912, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffe1934c280093c181c2b062a05e0908, server=037c58a19ddb,37267,1731751077844 2024-11-16T09:59:29,927 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffe1934c280093c181c2b062a05e0908, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T09:59:29,927 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731751162662.ffe1934c280093c181c2b062a05e0908., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 , the old value is region=TestAcidGuarantees,,1731751162662.ffe1934c280093c181c2b062a05e0908., hostname=037c58a19ddb,37267,1731751077844, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffe1934c280093c181c2b062a05e0908, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:59:29,927 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731751162662.ffe1934c280093c181c2b062a05e0908., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffe1934c280093c181c2b062a05e0908, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:59:29,927 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43418 deadline: 1731751229912, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffe1934c280093c181c2b062a05e0908, server=037c58a19ddb,37267,1731751077844 2024-11-16T09:59:29,927 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731751162662.ffe1934c280093c181c2b062a05e0908., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 because the exception is null or not the one we care about 2024-11-16T09:59:29,928 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731751162662.ffe1934c280093c181c2b062a05e0908., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 , the old value is region=TestAcidGuarantees,,1731751162662.ffe1934c280093c181c2b062a05e0908., hostname=037c58a19ddb,37267,1731751077844, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffe1934c280093c181c2b062a05e0908, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:59:29,928 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731751162662.ffe1934c280093c181c2b062a05e0908., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffe1934c280093c181c2b062a05e0908, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:59:29,928 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffe1934c280093c181c2b062a05e0908, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T09:59:29,928 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731751162662.ffe1934c280093c181c2b062a05e0908., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 because the exception is null or not the one we care about 2024-11-16T09:59:29,928 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43402 deadline: 1731751229913, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffe1934c280093c181c2b062a05e0908, server=037c58a19ddb,37267,1731751077844 2024-11-16T09:59:29,928 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731751162662.ffe1934c280093c181c2b062a05e0908., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 , the old value is region=TestAcidGuarantees,,1731751162662.ffe1934c280093c181c2b062a05e0908., hostname=037c58a19ddb,37267,1731751077844, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffe1934c280093c181c2b062a05e0908, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:59:29,928 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffe1934c280093c181c2b062a05e0908, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T09:59:29,928 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731751162662.ffe1934c280093c181c2b062a05e0908., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffe1934c280093c181c2b062a05e0908, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:59:29,928 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731751162662.ffe1934c280093c181c2b062a05e0908., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 because the exception is null or not the one we care about 2024-11-16T09:59:29,928 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43416 deadline: 1731751229914, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffe1934c280093c181c2b062a05e0908, server=037c58a19ddb,37267,1731751077844 2024-11-16T09:59:29,928 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731751162662.ffe1934c280093c181c2b062a05e0908., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 , the old value is region=TestAcidGuarantees,,1731751162662.ffe1934c280093c181c2b062a05e0908., hostname=037c58a19ddb,37267,1731751077844, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffe1934c280093c181c2b062a05e0908, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:59:29,928 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731751162662.ffe1934c280093c181c2b062a05e0908., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffe1934c280093c181c2b062a05e0908, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:59:29,928 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731751162662.ffe1934c280093c181c2b062a05e0908., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 because the exception is null or not the one we care about 2024-11-16T09:59:29,933 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37877 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=97 2024-11-16T09:59:29,933 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-16T09:59:29,934 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37877 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-16T09:59:29,935 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37877 {}] procedure2.ProcedureExecutor(1139): Stored pid=99, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=99, table=TestAcidGuarantees 2024-11-16T09:59:29,935 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37877 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=99 2024-11-16T09:59:29,935 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=99, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=99, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-16T09:59:29,936 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=99, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=99, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-16T09:59:29,936 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=100, ppid=99, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-16T09:59:30,028 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffe1934c280093c181c2b062a05e0908, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T09:59:30,029 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43332 deadline: 1731751230023, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffe1934c280093c181c2b062a05e0908, server=037c58a19ddb,37267,1731751077844 2024-11-16T09:59:30,029 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731751162662.ffe1934c280093c181c2b062a05e0908., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 , the old value is region=TestAcidGuarantees,,1731751162662.ffe1934c280093c181c2b062a05e0908., hostname=037c58a19ddb,37267,1731751077844, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffe1934c280093c181c2b062a05e0908, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:59:30,029 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731751162662.ffe1934c280093c181c2b062a05e0908., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffe1934c280093c181c2b062a05e0908, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:59:30,029 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731751162662.ffe1934c280093c181c2b062a05e0908., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 because the exception is null or not the one we care about 2024-11-16T09:59:30,040 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffe1934c280093c181c2b062a05e0908, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T09:59:30,040 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43362 deadline: 1731751230033, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffe1934c280093c181c2b062a05e0908, server=037c58a19ddb,37267,1731751077844 2024-11-16T09:59:30,040 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731751162662.ffe1934c280093c181c2b062a05e0908., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 , the old value is region=TestAcidGuarantees,,1731751162662.ffe1934c280093c181c2b062a05e0908., hostname=037c58a19ddb,37267,1731751077844, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffe1934c280093c181c2b062a05e0908, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:59:30,040 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffe1934c280093c181c2b062a05e0908, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T09:59:30,040 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731751162662.ffe1934c280093c181c2b062a05e0908., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffe1934c280093c181c2b062a05e0908, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:59:30,040 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731751162662.ffe1934c280093c181c2b062a05e0908., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 because the exception is null or not the one we care about 2024-11-16T09:59:30,040 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43418 deadline: 1731751230033, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffe1934c280093c181c2b062a05e0908, server=037c58a19ddb,37267,1731751077844 2024-11-16T09:59:30,040 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffe1934c280093c181c2b062a05e0908, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T09:59:30,040 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731751162662.ffe1934c280093c181c2b062a05e0908., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 , the old value is region=TestAcidGuarantees,,1731751162662.ffe1934c280093c181c2b062a05e0908., hostname=037c58a19ddb,37267,1731751077844, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffe1934c280093c181c2b062a05e0908, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:59:30,041 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43402 deadline: 1731751230033, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffe1934c280093c181c2b062a05e0908, server=037c58a19ddb,37267,1731751077844 2024-11-16T09:59:30,041 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731751162662.ffe1934c280093c181c2b062a05e0908., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffe1934c280093c181c2b062a05e0908, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:59:30,041 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731751162662.ffe1934c280093c181c2b062a05e0908., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 because the exception is null or not the one we care about 2024-11-16T09:59:30,041 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731751162662.ffe1934c280093c181c2b062a05e0908., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 , the old value is region=TestAcidGuarantees,,1731751162662.ffe1934c280093c181c2b062a05e0908., hostname=037c58a19ddb,37267,1731751077844, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffe1934c280093c181c2b062a05e0908, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:59:30,041 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffe1934c280093c181c2b062a05e0908, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T09:59:30,041 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731751162662.ffe1934c280093c181c2b062a05e0908., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffe1934c280093c181c2b062a05e0908, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:59:30,041 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731751162662.ffe1934c280093c181c2b062a05e0908., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 because the exception is null or not the one we care about 2024-11-16T09:59:30,041 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43416 deadline: 1731751230033, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffe1934c280093c181c2b062a05e0908, server=037c58a19ddb,37267,1731751077844 2024-11-16T09:59:30,041 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731751162662.ffe1934c280093c181c2b062a05e0908., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 , the old value is region=TestAcidGuarantees,,1731751162662.ffe1934c280093c181c2b062a05e0908., hostname=037c58a19ddb,37267,1731751077844, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffe1934c280093c181c2b062a05e0908, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:59:30,041 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731751162662.ffe1934c280093c181c2b062a05e0908., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffe1934c280093c181c2b062a05e0908, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:59:30,041 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731751162662.ffe1934c280093c181c2b062a05e0908., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 because the exception is null or not the one we care about 2024-11-16T09:59:30,042 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37877 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=99 2024-11-16T09:59:30,088 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=37267 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=100 2024-11-16T09:59:30,089 DEBUG [RS_FLUSH_OPERATIONS-regionserver/037c58a19ddb:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731751162662.ffe1934c280093c181c2b062a05e0908. 2024-11-16T09:59:30,089 DEBUG [RS_FLUSH_OPERATIONS-regionserver/037c58a19ddb:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731751162662.ffe1934c280093c181c2b062a05e0908. as already flushing 2024-11-16T09:59:30,089 DEBUG [RS_FLUSH_OPERATIONS-regionserver/037c58a19ddb:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731751162662.ffe1934c280093c181c2b062a05e0908. 2024-11-16T09:59:30,089 ERROR [RS_FLUSH_OPERATIONS-regionserver/037c58a19ddb:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] handler.RSProcedureHandler(58): pid=100 java.io.IOException: Unable to complete flush {ENCODED => ffe1934c280093c181c2b062a05e0908, NAME => 'TestAcidGuarantees,,1731751162662.ffe1934c280093c181c2b062a05e0908.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T09:59:30,089 DEBUG [RS_FLUSH_OPERATIONS-regionserver/037c58a19ddb:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=100 java.io.IOException: Unable to complete flush {ENCODED => ffe1934c280093c181c2b062a05e0908, NAME => 'TestAcidGuarantees,,1731751162662.ffe1934c280093c181c2b062a05e0908.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T09:59:30,092 DEBUG [RS:0;037c58a19ddb:37267-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/ffe1934c280093c181c2b062a05e0908/.tmp/B/be4ceea2e8b74c3fb5af6601e8cc803b as hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/ffe1934c280093c181c2b062a05e0908/B/be4ceea2e8b74c3fb5af6601e8cc803b 2024-11-16T09:59:30,092 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37877 {}] master.HMaster(4177): Remote procedure failed, pid=100 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ffe1934c280093c181c2b062a05e0908, NAME => 'TestAcidGuarantees,,1731751162662.ffe1934c280093c181c2b062a05e0908.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ffe1934c280093c181c2b062a05e0908, NAME => 'TestAcidGuarantees,,1731751162662.ffe1934c280093c181c2b062a05e0908.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T09:59:30,097 INFO [RS:0;037c58a19ddb:37267-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in ffe1934c280093c181c2b062a05e0908/B of ffe1934c280093c181c2b062a05e0908 into be4ceea2e8b74c3fb5af6601e8cc803b(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-16T09:59:30,097 DEBUG [RS:0;037c58a19ddb:37267-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for ffe1934c280093c181c2b062a05e0908: 2024-11-16T09:59:30,097 INFO [RS:0;037c58a19ddb:37267-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731751162662.ffe1934c280093c181c2b062a05e0908., storeName=ffe1934c280093c181c2b062a05e0908/B, priority=13, startTime=1731751169666; duration=0sec 2024-11-16T09:59:30,097 DEBUG [RS:0;037c58a19ddb:37267-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-16T09:59:30,097 DEBUG [RS:0;037c58a19ddb:37267-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: ffe1934c280093c181c2b062a05e0908:B 2024-11-16T09:59:30,097 DEBUG [RS:0;037c58a19ddb:37267-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-16T09:59:30,098 DEBUG [RS:0;037c58a19ddb:37267-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36003 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-16T09:59:30,098 DEBUG [RS:0;037c58a19ddb:37267-longCompactions-0 {}] regionserver.HStore(1541): ffe1934c280093c181c2b062a05e0908/C is initiating minor compaction (all files) 2024-11-16T09:59:30,098 INFO [RS:0;037c58a19ddb:37267-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of ffe1934c280093c181c2b062a05e0908/C in TestAcidGuarantees,,1731751162662.ffe1934c280093c181c2b062a05e0908. 2024-11-16T09:59:30,098 INFO [RS:0;037c58a19ddb:37267-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/ffe1934c280093c181c2b062a05e0908/C/5d8b2f5d76744a4a935a06e9f2b023ea, hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/ffe1934c280093c181c2b062a05e0908/C/9ad2541e56924229b4932313705f9787, hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/ffe1934c280093c181c2b062a05e0908/C/7fa927d3441046e0962c03ddbd3858b7] into tmpdir=hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/ffe1934c280093c181c2b062a05e0908/.tmp, totalSize=35.2 K 2024-11-16T09:59:30,099 DEBUG [RS:0;037c58a19ddb:37267-longCompactions-0 {}] compactions.Compactor(225): Compacting 5d8b2f5d76744a4a935a06e9f2b023ea, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=16, earliestPutTs=1731751165778 2024-11-16T09:59:30,099 DEBUG [RS:0;037c58a19ddb:37267-longCompactions-0 {}] compactions.Compactor(225): Compacting 9ad2541e56924229b4932313705f9787, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=40, earliestPutTs=1731751165822 2024-11-16T09:59:30,099 DEBUG [RS:0;037c58a19ddb:37267-longCompactions-0 {}] compactions.Compactor(225): Compacting 7fa927d3441046e0962c03ddbd3858b7, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=53, earliestPutTs=1731751166994 2024-11-16T09:59:30,104 DEBUG [RS:0;037c58a19ddb:37267-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/ffe1934c280093c181c2b062a05e0908/.tmp/A/4939aa1e225f4585a4c8405038aea298 as hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/ffe1934c280093c181c2b062a05e0908/A/4939aa1e225f4585a4c8405038aea298 2024-11-16T09:59:30,107 INFO [RS:0;037c58a19ddb:37267-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): ffe1934c280093c181c2b062a05e0908#C#compaction#311 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-16T09:59:30,108 DEBUG [RS:0;037c58a19ddb:37267-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/ffe1934c280093c181c2b062a05e0908/.tmp/C/ca4c37e2417b4a24b1ff87c6e2fd47f9 is 50, key is test_row_0/C:col10/1731751169177/Put/seqid=0 2024-11-16T09:59:30,110 INFO [RS:0;037c58a19ddb:37267-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in ffe1934c280093c181c2b062a05e0908/A of ffe1934c280093c181c2b062a05e0908 into 4939aa1e225f4585a4c8405038aea298(size=30.3 K), total size for store is 30.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-16T09:59:30,110 DEBUG [RS:0;037c58a19ddb:37267-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for ffe1934c280093c181c2b062a05e0908: 2024-11-16T09:59:30,110 INFO [RS:0;037c58a19ddb:37267-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731751162662.ffe1934c280093c181c2b062a05e0908., storeName=ffe1934c280093c181c2b062a05e0908/A, priority=13, startTime=1731751169666; duration=0sec 2024-11-16T09:59:30,110 DEBUG [RS:0;037c58a19ddb:37267-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-16T09:59:30,110 DEBUG [RS:0;037c58a19ddb:37267-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: ffe1934c280093c181c2b062a05e0908:A 2024-11-16T09:59:30,111 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37591 is added to blk_1073742187_1363 (size=12104) 2024-11-16T09:59:30,238 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffe1934c280093c181c2b062a05e0908, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T09:59:30,238 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43332 deadline: 1731751230232, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffe1934c280093c181c2b062a05e0908, server=037c58a19ddb,37267,1731751077844 2024-11-16T09:59:30,238 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731751162662.ffe1934c280093c181c2b062a05e0908., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 , the old value is region=TestAcidGuarantees,,1731751162662.ffe1934c280093c181c2b062a05e0908., hostname=037c58a19ddb,37267,1731751077844, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffe1934c280093c181c2b062a05e0908, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:59:30,238 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731751162662.ffe1934c280093c181c2b062a05e0908., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffe1934c280093c181c2b062a05e0908, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:59:30,238 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731751162662.ffe1934c280093c181c2b062a05e0908., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 because the exception is null or not the one we care about 2024-11-16T09:59:30,244 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=37267 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=100 2024-11-16T09:59:30,244 DEBUG [RS_FLUSH_OPERATIONS-regionserver/037c58a19ddb:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731751162662.ffe1934c280093c181c2b062a05e0908. 2024-11-16T09:59:30,245 DEBUG [RS_FLUSH_OPERATIONS-regionserver/037c58a19ddb:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731751162662.ffe1934c280093c181c2b062a05e0908. as already flushing 2024-11-16T09:59:30,245 DEBUG [RS_FLUSH_OPERATIONS-regionserver/037c58a19ddb:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731751162662.ffe1934c280093c181c2b062a05e0908. 2024-11-16T09:59:30,245 ERROR [RS_FLUSH_OPERATIONS-regionserver/037c58a19ddb:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] handler.RSProcedureHandler(58): pid=100 java.io.IOException: Unable to complete flush {ENCODED => ffe1934c280093c181c2b062a05e0908, NAME => 'TestAcidGuarantees,,1731751162662.ffe1934c280093c181c2b062a05e0908.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T09:59:30,245 DEBUG [RS_FLUSH_OPERATIONS-regionserver/037c58a19ddb:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=100 java.io.IOException: Unable to complete flush {ENCODED => ffe1934c280093c181c2b062a05e0908, NAME => 'TestAcidGuarantees,,1731751162662.ffe1934c280093c181c2b062a05e0908.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T09:59:30,245 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37877 {}] master.HMaster(4177): Remote procedure failed, pid=100 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ffe1934c280093c181c2b062a05e0908, NAME => 'TestAcidGuarantees,,1731751162662.ffe1934c280093c181c2b062a05e0908.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ffe1934c280093c181c2b062a05e0908, NAME => 'TestAcidGuarantees,,1731751162662.ffe1934c280093c181c2b062a05e0908.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T09:59:30,248 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffe1934c280093c181c2b062a05e0908, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T09:59:30,249 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43362 deadline: 1731751230243, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffe1934c280093c181c2b062a05e0908, server=037c58a19ddb,37267,1731751077844 2024-11-16T09:59:30,249 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffe1934c280093c181c2b062a05e0908, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T09:59:30,249 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731751162662.ffe1934c280093c181c2b062a05e0908., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 , the old value is region=TestAcidGuarantees,,1731751162662.ffe1934c280093c181c2b062a05e0908., hostname=037c58a19ddb,37267,1731751077844, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffe1934c280093c181c2b062a05e0908, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:59:30,249 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731751162662.ffe1934c280093c181c2b062a05e0908., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffe1934c280093c181c2b062a05e0908, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:59:30,249 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43402 deadline: 1731751230243, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffe1934c280093c181c2b062a05e0908, server=037c58a19ddb,37267,1731751077844 2024-11-16T09:59:30,249 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731751162662.ffe1934c280093c181c2b062a05e0908., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 because the exception is null or not the one we care about 2024-11-16T09:59:30,249 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffe1934c280093c181c2b062a05e0908, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T09:59:30,249 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731751162662.ffe1934c280093c181c2b062a05e0908., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 , the old value is region=TestAcidGuarantees,,1731751162662.ffe1934c280093c181c2b062a05e0908., hostname=037c58a19ddb,37267,1731751077844, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffe1934c280093c181c2b062a05e0908, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:59:30,249 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43416 deadline: 1731751230243, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffe1934c280093c181c2b062a05e0908, server=037c58a19ddb,37267,1731751077844 2024-11-16T09:59:30,249 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731751162662.ffe1934c280093c181c2b062a05e0908., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffe1934c280093c181c2b062a05e0908, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:59:30,249 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731751162662.ffe1934c280093c181c2b062a05e0908., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 because the exception is null or not the one we care about 2024-11-16T09:59:30,249 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731751162662.ffe1934c280093c181c2b062a05e0908., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 , the old value is region=TestAcidGuarantees,,1731751162662.ffe1934c280093c181c2b062a05e0908., hostname=037c58a19ddb,37267,1731751077844, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffe1934c280093c181c2b062a05e0908, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:59:30,249 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731751162662.ffe1934c280093c181c2b062a05e0908., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffe1934c280093c181c2b062a05e0908, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:59:30,249 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731751162662.ffe1934c280093c181c2b062a05e0908., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 because the exception is null or not the one we care about 2024-11-16T09:59:30,253 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37877 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=99 2024-11-16T09:59:30,259 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffe1934c280093c181c2b062a05e0908, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T09:59:30,259 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43418 deadline: 1731751230253, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffe1934c280093c181c2b062a05e0908, server=037c58a19ddb,37267,1731751077844 2024-11-16T09:59:30,259 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731751162662.ffe1934c280093c181c2b062a05e0908., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 , the old value is region=TestAcidGuarantees,,1731751162662.ffe1934c280093c181c2b062a05e0908., hostname=037c58a19ddb,37267,1731751077844, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffe1934c280093c181c2b062a05e0908, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:59:30,259 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731751162662.ffe1934c280093c181c2b062a05e0908., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffe1934c280093c181c2b062a05e0908, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:59:30,259 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731751162662.ffe1934c280093c181c2b062a05e0908., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 because the exception is null or not the one we care about 2024-11-16T09:59:30,308 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T09:59:30,312 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411166857ff57a2f341f699ea264dea73761a_ffe1934c280093c181c2b062a05e0908 to hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411166857ff57a2f341f699ea264dea73761a_ffe1934c280093c181c2b062a05e0908 2024-11-16T09:59:30,313 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/ffe1934c280093c181c2b062a05e0908/.tmp/A/6d3471f357e64c24aa32cc3a11602671, store: [table=TestAcidGuarantees family=A region=ffe1934c280093c181c2b062a05e0908] 2024-11-16T09:59:30,313 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/ffe1934c280093c181c2b062a05e0908/.tmp/A/6d3471f357e64c24aa32cc3a11602671 is 175, key is test_row_0/A:col10/1731751169896/Put/seqid=0 2024-11-16T09:59:30,316 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37591 is added to blk_1073742188_1364 (size=39549) 2024-11-16T09:59:30,396 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=37267 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=100 2024-11-16T09:59:30,397 DEBUG [RS_FLUSH_OPERATIONS-regionserver/037c58a19ddb:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731751162662.ffe1934c280093c181c2b062a05e0908. 2024-11-16T09:59:30,397 DEBUG [RS_FLUSH_OPERATIONS-regionserver/037c58a19ddb:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731751162662.ffe1934c280093c181c2b062a05e0908. as already flushing 2024-11-16T09:59:30,397 DEBUG [RS_FLUSH_OPERATIONS-regionserver/037c58a19ddb:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731751162662.ffe1934c280093c181c2b062a05e0908. 2024-11-16T09:59:30,397 ERROR [RS_FLUSH_OPERATIONS-regionserver/037c58a19ddb:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] handler.RSProcedureHandler(58): pid=100 java.io.IOException: Unable to complete flush {ENCODED => ffe1934c280093c181c2b062a05e0908, NAME => 'TestAcidGuarantees,,1731751162662.ffe1934c280093c181c2b062a05e0908.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T09:59:30,397 DEBUG [RS_FLUSH_OPERATIONS-regionserver/037c58a19ddb:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=100 java.io.IOException: Unable to complete flush {ENCODED => ffe1934c280093c181c2b062a05e0908, NAME => 'TestAcidGuarantees,,1731751162662.ffe1934c280093c181c2b062a05e0908.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T09:59:30,398 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37877 {}] master.HMaster(4177): Remote procedure failed, pid=100 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ffe1934c280093c181c2b062a05e0908, NAME => 'TestAcidGuarantees,,1731751162662.ffe1934c280093c181c2b062a05e0908.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ffe1934c280093c181c2b062a05e0908, NAME => 'TestAcidGuarantees,,1731751162662.ffe1934c280093c181c2b062a05e0908.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T09:59:30,517 DEBUG [RS:0;037c58a19ddb:37267-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/ffe1934c280093c181c2b062a05e0908/.tmp/C/ca4c37e2417b4a24b1ff87c6e2fd47f9 as hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/ffe1934c280093c181c2b062a05e0908/C/ca4c37e2417b4a24b1ff87c6e2fd47f9 2024-11-16T09:59:30,522 INFO [RS:0;037c58a19ddb:37267-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in ffe1934c280093c181c2b062a05e0908/C of ffe1934c280093c181c2b062a05e0908 into ca4c37e2417b4a24b1ff87c6e2fd47f9(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-16T09:59:30,522 DEBUG [RS:0;037c58a19ddb:37267-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for ffe1934c280093c181c2b062a05e0908: 2024-11-16T09:59:30,522 INFO [RS:0;037c58a19ddb:37267-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731751162662.ffe1934c280093c181c2b062a05e0908., storeName=ffe1934c280093c181c2b062a05e0908/C, priority=13, startTime=1731751169666; duration=0sec 2024-11-16T09:59:30,523 DEBUG [RS:0;037c58a19ddb:37267-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-16T09:59:30,523 DEBUG [RS:0;037c58a19ddb:37267-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: ffe1934c280093c181c2b062a05e0908:C 2024-11-16T09:59:30,549 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffe1934c280093c181c2b062a05e0908, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T09:59:30,549 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43332 deadline: 1731751230543, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffe1934c280093c181c2b062a05e0908, server=037c58a19ddb,37267,1731751077844 2024-11-16T09:59:30,550 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=37267 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=100 2024-11-16T09:59:30,550 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731751162662.ffe1934c280093c181c2b062a05e0908., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 , the old value is region=TestAcidGuarantees,,1731751162662.ffe1934c280093c181c2b062a05e0908., hostname=037c58a19ddb,37267,1731751077844, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffe1934c280093c181c2b062a05e0908, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:59:30,550 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731751162662.ffe1934c280093c181c2b062a05e0908., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffe1934c280093c181c2b062a05e0908, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:59:30,550 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731751162662.ffe1934c280093c181c2b062a05e0908., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 because the exception is null or not the one we care about 2024-11-16T09:59:30,550 DEBUG [RS_FLUSH_OPERATIONS-regionserver/037c58a19ddb:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731751162662.ffe1934c280093c181c2b062a05e0908. 2024-11-16T09:59:30,550 DEBUG [RS_FLUSH_OPERATIONS-regionserver/037c58a19ddb:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731751162662.ffe1934c280093c181c2b062a05e0908. as already flushing 2024-11-16T09:59:30,550 DEBUG [RS_FLUSH_OPERATIONS-regionserver/037c58a19ddb:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731751162662.ffe1934c280093c181c2b062a05e0908. 2024-11-16T09:59:30,550 ERROR [RS_FLUSH_OPERATIONS-regionserver/037c58a19ddb:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] handler.RSProcedureHandler(58): pid=100 java.io.IOException: Unable to complete flush {ENCODED => ffe1934c280093c181c2b062a05e0908, NAME => 'TestAcidGuarantees,,1731751162662.ffe1934c280093c181c2b062a05e0908.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T09:59:30,550 DEBUG [RS_FLUSH_OPERATIONS-regionserver/037c58a19ddb:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=100 java.io.IOException: Unable to complete flush {ENCODED => ffe1934c280093c181c2b062a05e0908, NAME => 'TestAcidGuarantees,,1731751162662.ffe1934c280093c181c2b062a05e0908.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T09:59:30,551 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37877 {}] master.HMaster(4177): Remote procedure failed, pid=100 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ffe1934c280093c181c2b062a05e0908, NAME => 'TestAcidGuarantees,,1731751162662.ffe1934c280093c181c2b062a05e0908.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ffe1934c280093c181c2b062a05e0908, NAME => 'TestAcidGuarantees,,1731751162662.ffe1934c280093c181c2b062a05e0908.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T09:59:30,559 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffe1934c280093c181c2b062a05e0908, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T09:59:30,560 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43362 deadline: 1731751230553, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffe1934c280093c181c2b062a05e0908, server=037c58a19ddb,37267,1731751077844 2024-11-16T09:59:30,560 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffe1934c280093c181c2b062a05e0908, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T09:59:30,560 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731751162662.ffe1934c280093c181c2b062a05e0908., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 , the old value is region=TestAcidGuarantees,,1731751162662.ffe1934c280093c181c2b062a05e0908., hostname=037c58a19ddb,37267,1731751077844, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffe1934c280093c181c2b062a05e0908, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:59:30,560 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731751162662.ffe1934c280093c181c2b062a05e0908., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffe1934c280093c181c2b062a05e0908, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:59:30,560 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43402 deadline: 1731751230553, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffe1934c280093c181c2b062a05e0908, server=037c58a19ddb,37267,1731751077844 2024-11-16T09:59:30,560 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731751162662.ffe1934c280093c181c2b062a05e0908., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 because the exception is null or not the one we care about 2024-11-16T09:59:30,560 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731751162662.ffe1934c280093c181c2b062a05e0908., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 , the old value is region=TestAcidGuarantees,,1731751162662.ffe1934c280093c181c2b062a05e0908., hostname=037c58a19ddb,37267,1731751077844, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffe1934c280093c181c2b062a05e0908, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:59:30,560 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731751162662.ffe1934c280093c181c2b062a05e0908., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffe1934c280093c181c2b062a05e0908, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:59:30,560 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731751162662.ffe1934c280093c181c2b062a05e0908., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 because the exception is null or not the one we care about 2024-11-16T09:59:30,563 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37877 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=99 2024-11-16T09:59:30,570 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffe1934c280093c181c2b062a05e0908, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T09:59:30,571 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43416 deadline: 1731751230563, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffe1934c280093c181c2b062a05e0908, server=037c58a19ddb,37267,1731751077844 2024-11-16T09:59:30,571 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffe1934c280093c181c2b062a05e0908, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T09:59:30,571 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731751162662.ffe1934c280093c181c2b062a05e0908., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 , the old value is region=TestAcidGuarantees,,1731751162662.ffe1934c280093c181c2b062a05e0908., hostname=037c58a19ddb,37267,1731751077844, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffe1934c280093c181c2b062a05e0908, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:59:30,571 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43418 deadline: 1731751230563, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffe1934c280093c181c2b062a05e0908, server=037c58a19ddb,37267,1731751077844 2024-11-16T09:59:30,571 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731751162662.ffe1934c280093c181c2b062a05e0908., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffe1934c280093c181c2b062a05e0908, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:59:30,571 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731751162662.ffe1934c280093c181c2b062a05e0908., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 because the exception is null or not the one we care about 2024-11-16T09:59:30,571 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731751162662.ffe1934c280093c181c2b062a05e0908., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 , the old value is region=TestAcidGuarantees,,1731751162662.ffe1934c280093c181c2b062a05e0908., hostname=037c58a19ddb,37267,1731751077844, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffe1934c280093c181c2b062a05e0908, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:59:30,571 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731751162662.ffe1934c280093c181c2b062a05e0908., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffe1934c280093c181c2b062a05e0908, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:59:30,571 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731751162662.ffe1934c280093c181c2b062a05e0908., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 because the exception is null or not the one we care about 2024-11-16T09:59:30,703 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=37267 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=100 2024-11-16T09:59:30,703 DEBUG [RS_FLUSH_OPERATIONS-regionserver/037c58a19ddb:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731751162662.ffe1934c280093c181c2b062a05e0908. 2024-11-16T09:59:30,703 DEBUG [RS_FLUSH_OPERATIONS-regionserver/037c58a19ddb:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731751162662.ffe1934c280093c181c2b062a05e0908. as already flushing 2024-11-16T09:59:30,703 DEBUG [RS_FLUSH_OPERATIONS-regionserver/037c58a19ddb:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731751162662.ffe1934c280093c181c2b062a05e0908. 2024-11-16T09:59:30,703 ERROR [RS_FLUSH_OPERATIONS-regionserver/037c58a19ddb:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] handler.RSProcedureHandler(58): pid=100 java.io.IOException: Unable to complete flush {ENCODED => ffe1934c280093c181c2b062a05e0908, NAME => 'TestAcidGuarantees,,1731751162662.ffe1934c280093c181c2b062a05e0908.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T09:59:30,703 DEBUG [RS_FLUSH_OPERATIONS-regionserver/037c58a19ddb:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=100 java.io.IOException: Unable to complete flush {ENCODED => ffe1934c280093c181c2b062a05e0908, NAME => 'TestAcidGuarantees,,1731751162662.ffe1934c280093c181c2b062a05e0908.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T09:59:30,704 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37877 {}] master.HMaster(4177): Remote procedure failed, pid=100 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ffe1934c280093c181c2b062a05e0908, NAME => 'TestAcidGuarantees,,1731751162662.ffe1934c280093c181c2b062a05e0908.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ffe1934c280093c181c2b062a05e0908, NAME => 'TestAcidGuarantees,,1731751162662.ffe1934c280093c181c2b062a05e0908.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T09:59:30,717 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=77, memsize=47.0 K, hasBloomFilter=true, into tmp file hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/ffe1934c280093c181c2b062a05e0908/.tmp/A/6d3471f357e64c24aa32cc3a11602671 2024-11-16T09:59:30,725 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/ffe1934c280093c181c2b062a05e0908/.tmp/B/d6f737504c674d9397fba019aa8885a5 is 50, key is test_row_0/B:col10/1731751169896/Put/seqid=0 2024-11-16T09:59:30,729 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37591 is added to blk_1073742189_1365 (size=12001) 2024-11-16T09:59:30,856 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=37267 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=100 2024-11-16T09:59:30,856 DEBUG [RS_FLUSH_OPERATIONS-regionserver/037c58a19ddb:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731751162662.ffe1934c280093c181c2b062a05e0908. 2024-11-16T09:59:30,856 DEBUG [RS_FLUSH_OPERATIONS-regionserver/037c58a19ddb:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731751162662.ffe1934c280093c181c2b062a05e0908. as already flushing 2024-11-16T09:59:30,857 DEBUG [RS_FLUSH_OPERATIONS-regionserver/037c58a19ddb:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731751162662.ffe1934c280093c181c2b062a05e0908. 2024-11-16T09:59:30,857 ERROR [RS_FLUSH_OPERATIONS-regionserver/037c58a19ddb:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] handler.RSProcedureHandler(58): pid=100 java.io.IOException: Unable to complete flush {ENCODED => ffe1934c280093c181c2b062a05e0908, NAME => 'TestAcidGuarantees,,1731751162662.ffe1934c280093c181c2b062a05e0908.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T09:59:30,857 DEBUG [RS_FLUSH_OPERATIONS-regionserver/037c58a19ddb:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=100 java.io.IOException: Unable to complete flush {ENCODED => ffe1934c280093c181c2b062a05e0908, NAME => 'TestAcidGuarantees,,1731751162662.ffe1934c280093c181c2b062a05e0908.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T09:59:30,858 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37877 {}] master.HMaster(4177): Remote procedure failed, pid=100 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ffe1934c280093c181c2b062a05e0908, NAME => 'TestAcidGuarantees,,1731751162662.ffe1934c280093c181c2b062a05e0908.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ffe1934c280093c181c2b062a05e0908, NAME => 'TestAcidGuarantees,,1731751162662.ffe1934c280093c181c2b062a05e0908.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T09:59:31,010 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=37267 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=100 2024-11-16T09:59:31,010 DEBUG [RS_FLUSH_OPERATIONS-regionserver/037c58a19ddb:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731751162662.ffe1934c280093c181c2b062a05e0908. 2024-11-16T09:59:31,010 DEBUG [RS_FLUSH_OPERATIONS-regionserver/037c58a19ddb:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731751162662.ffe1934c280093c181c2b062a05e0908. as already flushing 2024-11-16T09:59:31,010 DEBUG [RS_FLUSH_OPERATIONS-regionserver/037c58a19ddb:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731751162662.ffe1934c280093c181c2b062a05e0908. 2024-11-16T09:59:31,010 ERROR [RS_FLUSH_OPERATIONS-regionserver/037c58a19ddb:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] handler.RSProcedureHandler(58): pid=100 java.io.IOException: Unable to complete flush {ENCODED => ffe1934c280093c181c2b062a05e0908, NAME => 'TestAcidGuarantees,,1731751162662.ffe1934c280093c181c2b062a05e0908.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T09:59:31,010 DEBUG [RS_FLUSH_OPERATIONS-regionserver/037c58a19ddb:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=100 java.io.IOException: Unable to complete flush {ENCODED => ffe1934c280093c181c2b062a05e0908, NAME => 'TestAcidGuarantees,,1731751162662.ffe1934c280093c181c2b062a05e0908.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T09:59:31,011 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37877 {}] master.HMaster(4177): Remote procedure failed, pid=100 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ffe1934c280093c181c2b062a05e0908, NAME => 'TestAcidGuarantees,,1731751162662.ffe1934c280093c181c2b062a05e0908.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ffe1934c280093c181c2b062a05e0908, NAME => 'TestAcidGuarantees,,1731751162662.ffe1934c280093c181c2b062a05e0908.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T09:59:31,068 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffe1934c280093c181c2b062a05e0908, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T09:59:31,069 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43332 deadline: 1731751231063, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffe1934c280093c181c2b062a05e0908, server=037c58a19ddb,37267,1731751077844 2024-11-16T09:59:31,069 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731751162662.ffe1934c280093c181c2b062a05e0908., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 , the old value is region=TestAcidGuarantees,,1731751162662.ffe1934c280093c181c2b062a05e0908., hostname=037c58a19ddb,37267,1731751077844, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffe1934c280093c181c2b062a05e0908, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:59:31,069 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731751162662.ffe1934c280093c181c2b062a05e0908., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffe1934c280093c181c2b062a05e0908, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:59:31,069 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731751162662.ffe1934c280093c181c2b062a05e0908., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 because the exception is null or not the one we care about 2024-11-16T09:59:31,073 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37877 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=99 2024-11-16T09:59:31,078 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffe1934c280093c181c2b062a05e0908, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T09:59:31,078 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43362 deadline: 1731751231072, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffe1934c280093c181c2b062a05e0908, server=037c58a19ddb,37267,1731751077844 2024-11-16T09:59:31,078 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffe1934c280093c181c2b062a05e0908, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T09:59:31,079 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731751162662.ffe1934c280093c181c2b062a05e0908., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 , the old value is region=TestAcidGuarantees,,1731751162662.ffe1934c280093c181c2b062a05e0908., hostname=037c58a19ddb,37267,1731751077844, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffe1934c280093c181c2b062a05e0908, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:59:31,079 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43402 deadline: 1731751231072, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffe1934c280093c181c2b062a05e0908, server=037c58a19ddb,37267,1731751077844 2024-11-16T09:59:31,079 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731751162662.ffe1934c280093c181c2b062a05e0908., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffe1934c280093c181c2b062a05e0908, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:59:31,079 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731751162662.ffe1934c280093c181c2b062a05e0908., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 because the exception is null or not the one we care about 2024-11-16T09:59:31,079 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731751162662.ffe1934c280093c181c2b062a05e0908., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 , the old value is region=TestAcidGuarantees,,1731751162662.ffe1934c280093c181c2b062a05e0908., hostname=037c58a19ddb,37267,1731751077844, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffe1934c280093c181c2b062a05e0908, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:59:31,079 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731751162662.ffe1934c280093c181c2b062a05e0908., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffe1934c280093c181c2b062a05e0908, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:59:31,079 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731751162662.ffe1934c280093c181c2b062a05e0908., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 because the exception is null or not the one we care about 2024-11-16T09:59:31,088 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffe1934c280093c181c2b062a05e0908, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T09:59:31,088 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43416 deadline: 1731751231083, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffe1934c280093c181c2b062a05e0908, server=037c58a19ddb,37267,1731751077844 2024-11-16T09:59:31,089 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffe1934c280093c181c2b062a05e0908, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T09:59:31,089 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731751162662.ffe1934c280093c181c2b062a05e0908., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 , the old value is region=TestAcidGuarantees,,1731751162662.ffe1934c280093c181c2b062a05e0908., hostname=037c58a19ddb,37267,1731751077844, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffe1934c280093c181c2b062a05e0908, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:59:31,089 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43418 deadline: 1731751231083, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffe1934c280093c181c2b062a05e0908, server=037c58a19ddb,37267,1731751077844 2024-11-16T09:59:31,089 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731751162662.ffe1934c280093c181c2b062a05e0908., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffe1934c280093c181c2b062a05e0908, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:59:31,089 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731751162662.ffe1934c280093c181c2b062a05e0908., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 because the exception is null or not the one we care about 2024-11-16T09:59:31,089 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731751162662.ffe1934c280093c181c2b062a05e0908., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 , the old value is region=TestAcidGuarantees,,1731751162662.ffe1934c280093c181c2b062a05e0908., hostname=037c58a19ddb,37267,1731751077844, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffe1934c280093c181c2b062a05e0908, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:59:31,089 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731751162662.ffe1934c280093c181c2b062a05e0908., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffe1934c280093c181c2b062a05e0908, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:59:31,089 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731751162662.ffe1934c280093c181c2b062a05e0908., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 because the exception is null or not the one we care about 2024-11-16T09:59:31,130 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=77 (bloomFilter=true), to=hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/ffe1934c280093c181c2b062a05e0908/.tmp/B/d6f737504c674d9397fba019aa8885a5 2024-11-16T09:59:31,137 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/ffe1934c280093c181c2b062a05e0908/.tmp/C/42861dc2507c4c0f8d9b4b330a334f9b is 50, key is test_row_0/C:col10/1731751169896/Put/seqid=0 2024-11-16T09:59:31,141 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37591 is added to blk_1073742190_1366 (size=12001) 2024-11-16T09:59:31,141 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=77 (bloomFilter=true), to=hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/ffe1934c280093c181c2b062a05e0908/.tmp/C/42861dc2507c4c0f8d9b4b330a334f9b 2024-11-16T09:59:31,145 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/ffe1934c280093c181c2b062a05e0908/.tmp/A/6d3471f357e64c24aa32cc3a11602671 as hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/ffe1934c280093c181c2b062a05e0908/A/6d3471f357e64c24aa32cc3a11602671 2024-11-16T09:59:31,148 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/ffe1934c280093c181c2b062a05e0908/A/6d3471f357e64c24aa32cc3a11602671, entries=200, sequenceid=77, filesize=38.6 K 2024-11-16T09:59:31,149 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/ffe1934c280093c181c2b062a05e0908/.tmp/B/d6f737504c674d9397fba019aa8885a5 as hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/ffe1934c280093c181c2b062a05e0908/B/d6f737504c674d9397fba019aa8885a5 2024-11-16T09:59:31,153 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/ffe1934c280093c181c2b062a05e0908/B/d6f737504c674d9397fba019aa8885a5, entries=150, sequenceid=77, filesize=11.7 K 2024-11-16T09:59:31,154 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/ffe1934c280093c181c2b062a05e0908/.tmp/C/42861dc2507c4c0f8d9b4b330a334f9b as hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/ffe1934c280093c181c2b062a05e0908/C/42861dc2507c4c0f8d9b4b330a334f9b 2024-11-16T09:59:31,157 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/ffe1934c280093c181c2b062a05e0908/C/42861dc2507c4c0f8d9b4b330a334f9b, entries=150, sequenceid=77, filesize=11.7 K 2024-11-16T09:59:31,158 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~140.89 KB/144270, heapSize ~369.84 KB/378720, currentSize=60.38 KB/61830 for ffe1934c280093c181c2b062a05e0908 in 1261ms, sequenceid=77, compaction requested=false 2024-11-16T09:59:31,158 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for ffe1934c280093c181c2b062a05e0908: 2024-11-16T09:59:31,163 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=37267 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=100 2024-11-16T09:59:31,163 DEBUG [RS_FLUSH_OPERATIONS-regionserver/037c58a19ddb:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731751162662.ffe1934c280093c181c2b062a05e0908. 2024-11-16T09:59:31,163 INFO [RS_FLUSH_OPERATIONS-regionserver/037c58a19ddb:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HRegion(2902): Flushing ffe1934c280093c181c2b062a05e0908 3/3 column families, dataSize=60.38 KB heapSize=158.95 KB 2024-11-16T09:59:31,170 DEBUG [RS_FLUSH_OPERATIONS-regionserver/037c58a19ddb:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241116beb0e73d2beb4d2f872976e5b9933168_ffe1934c280093c181c2b062a05e0908 is 50, key is test_row_0/A:col10/1731751169909/Put/seqid=0 2024-11-16T09:59:31,174 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37591 is added to blk_1073742191_1367 (size=12154) 2024-11-16T09:59:31,575 DEBUG [RS_FLUSH_OPERATIONS-regionserver/037c58a19ddb:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T09:59:31,579 INFO [RS_FLUSH_OPERATIONS-regionserver/037c58a19ddb:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241116beb0e73d2beb4d2f872976e5b9933168_ffe1934c280093c181c2b062a05e0908 to hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241116beb0e73d2beb4d2f872976e5b9933168_ffe1934c280093c181c2b062a05e0908 2024-11-16T09:59:31,579 DEBUG [RS_FLUSH_OPERATIONS-regionserver/037c58a19ddb:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/ffe1934c280093c181c2b062a05e0908/.tmp/A/70b600411963462196c33bf15fa80c47, store: [table=TestAcidGuarantees family=A region=ffe1934c280093c181c2b062a05e0908] 2024-11-16T09:59:31,580 DEBUG [RS_FLUSH_OPERATIONS-regionserver/037c58a19ddb:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/ffe1934c280093c181c2b062a05e0908/.tmp/A/70b600411963462196c33bf15fa80c47 is 175, key is test_row_0/A:col10/1731751169909/Put/seqid=0 2024-11-16T09:59:31,584 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37591 is added to blk_1073742192_1368 (size=30955) 2024-11-16T09:59:31,985 INFO [RS_FLUSH_OPERATIONS-regionserver/037c58a19ddb:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=92, memsize=20.1 K, hasBloomFilter=true, into tmp file hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/ffe1934c280093c181c2b062a05e0908/.tmp/A/70b600411963462196c33bf15fa80c47 2024-11-16T09:59:31,994 DEBUG [RS_FLUSH_OPERATIONS-regionserver/037c58a19ddb:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/ffe1934c280093c181c2b062a05e0908/.tmp/B/80fb85a1b63c419f8a6bc5905300e5f5 is 50, key is test_row_0/B:col10/1731751169909/Put/seqid=0 2024-11-16T09:59:31,998 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37591 is added to blk_1073742193_1369 (size=12001) 2024-11-16T09:59:32,083 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37877 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=99 2024-11-16T09:59:32,087 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] regionserver.HRegion(8855): Flush requested on ffe1934c280093c181c2b062a05e0908 2024-11-16T09:59:32,087 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731751162662.ffe1934c280093c181c2b062a05e0908. as already flushing 2024-11-16T09:59:32,173 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffe1934c280093c181c2b062a05e0908, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T09:59:32,173 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffe1934c280093c181c2b062a05e0908, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T09:59:32,173 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43362 deadline: 1731751232152, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffe1934c280093c181c2b062a05e0908, server=037c58a19ddb,37267,1731751077844 2024-11-16T09:59:32,173 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43402 deadline: 1731751232151, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffe1934c280093c181c2b062a05e0908, server=037c58a19ddb,37267,1731751077844 2024-11-16T09:59:32,173 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffe1934c280093c181c2b062a05e0908, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T09:59:32,173 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43418 deadline: 1731751232153, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffe1934c280093c181c2b062a05e0908, server=037c58a19ddb,37267,1731751077844 2024-11-16T09:59:32,173 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731751162662.ffe1934c280093c181c2b062a05e0908., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 , the old value is region=TestAcidGuarantees,,1731751162662.ffe1934c280093c181c2b062a05e0908., hostname=037c58a19ddb,37267,1731751077844, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffe1934c280093c181c2b062a05e0908, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:59:32,173 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731751162662.ffe1934c280093c181c2b062a05e0908., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffe1934c280093c181c2b062a05e0908, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:59:32,173 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731751162662.ffe1934c280093c181c2b062a05e0908., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 , the old value is region=TestAcidGuarantees,,1731751162662.ffe1934c280093c181c2b062a05e0908., hostname=037c58a19ddb,37267,1731751077844, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffe1934c280093c181c2b062a05e0908, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:59:32,173 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731751162662.ffe1934c280093c181c2b062a05e0908., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 because the exception is null or not the one we care about 2024-11-16T09:59:32,173 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731751162662.ffe1934c280093c181c2b062a05e0908., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffe1934c280093c181c2b062a05e0908, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:59:32,173 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731751162662.ffe1934c280093c181c2b062a05e0908., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 because the exception is null or not the one we care about 2024-11-16T09:59:32,174 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731751162662.ffe1934c280093c181c2b062a05e0908., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 , the old value is region=TestAcidGuarantees,,1731751162662.ffe1934c280093c181c2b062a05e0908., hostname=037c58a19ddb,37267,1731751077844, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffe1934c280093c181c2b062a05e0908, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:59:32,174 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731751162662.ffe1934c280093c181c2b062a05e0908., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffe1934c280093c181c2b062a05e0908, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:59:32,174 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731751162662.ffe1934c280093c181c2b062a05e0908., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 because the exception is null or not the one we care about 2024-11-16T09:59:32,184 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffe1934c280093c181c2b062a05e0908, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T09:59:32,184 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43332 deadline: 1731751232173, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffe1934c280093c181c2b062a05e0908, server=037c58a19ddb,37267,1731751077844 2024-11-16T09:59:32,184 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731751162662.ffe1934c280093c181c2b062a05e0908., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 , the old value is region=TestAcidGuarantees,,1731751162662.ffe1934c280093c181c2b062a05e0908., hostname=037c58a19ddb,37267,1731751077844, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffe1934c280093c181c2b062a05e0908, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:59:32,184 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731751162662.ffe1934c280093c181c2b062a05e0908., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffe1934c280093c181c2b062a05e0908, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:59:32,184 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731751162662.ffe1934c280093c181c2b062a05e0908., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 because the exception is null or not the one we care about 2024-11-16T09:59:32,184 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffe1934c280093c181c2b062a05e0908, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T09:59:32,184 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43416 deadline: 1731751232173, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffe1934c280093c181c2b062a05e0908, server=037c58a19ddb,37267,1731751077844 2024-11-16T09:59:32,185 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731751162662.ffe1934c280093c181c2b062a05e0908., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 , the old value is region=TestAcidGuarantees,,1731751162662.ffe1934c280093c181c2b062a05e0908., hostname=037c58a19ddb,37267,1731751077844, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffe1934c280093c181c2b062a05e0908, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:59:32,185 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731751162662.ffe1934c280093c181c2b062a05e0908., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffe1934c280093c181c2b062a05e0908, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:59:32,185 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731751162662.ffe1934c280093c181c2b062a05e0908., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 because the exception is null or not the one we care about 2024-11-16T09:59:32,292 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffe1934c280093c181c2b062a05e0908, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T09:59:32,292 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43362 deadline: 1731751232283, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffe1934c280093c181c2b062a05e0908, server=037c58a19ddb,37267,1731751077844 2024-11-16T09:59:32,292 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffe1934c280093c181c2b062a05e0908, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T09:59:32,292 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43402 deadline: 1731751232283, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffe1934c280093c181c2b062a05e0908, server=037c58a19ddb,37267,1731751077844 2024-11-16T09:59:32,292 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731751162662.ffe1934c280093c181c2b062a05e0908., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 , the old value is region=TestAcidGuarantees,,1731751162662.ffe1934c280093c181c2b062a05e0908., hostname=037c58a19ddb,37267,1731751077844, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffe1934c280093c181c2b062a05e0908, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:59:32,292 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731751162662.ffe1934c280093c181c2b062a05e0908., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffe1934c280093c181c2b062a05e0908, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:59:32,292 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731751162662.ffe1934c280093c181c2b062a05e0908., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 because the exception is null or not the one we care about 2024-11-16T09:59:32,292 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffe1934c280093c181c2b062a05e0908, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T09:59:32,292 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731751162662.ffe1934c280093c181c2b062a05e0908., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 , the old value is region=TestAcidGuarantees,,1731751162662.ffe1934c280093c181c2b062a05e0908., hostname=037c58a19ddb,37267,1731751077844, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffe1934c280093c181c2b062a05e0908, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:59:32,292 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731751162662.ffe1934c280093c181c2b062a05e0908., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffe1934c280093c181c2b062a05e0908, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:59:32,292 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731751162662.ffe1934c280093c181c2b062a05e0908., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 because the exception is null or not the one we care about 2024-11-16T09:59:32,292 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43418 deadline: 1731751232283, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffe1934c280093c181c2b062a05e0908, server=037c58a19ddb,37267,1731751077844 2024-11-16T09:59:32,293 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731751162662.ffe1934c280093c181c2b062a05e0908., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 , the old value is region=TestAcidGuarantees,,1731751162662.ffe1934c280093c181c2b062a05e0908., hostname=037c58a19ddb,37267,1731751077844, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffe1934c280093c181c2b062a05e0908, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:59:32,293 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731751162662.ffe1934c280093c181c2b062a05e0908., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffe1934c280093c181c2b062a05e0908, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:59:32,293 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731751162662.ffe1934c280093c181c2b062a05e0908., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 because the exception is null or not the one we care about 2024-11-16T09:59:32,296 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffe1934c280093c181c2b062a05e0908, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T09:59:32,296 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43332 deadline: 1731751232293, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffe1934c280093c181c2b062a05e0908, server=037c58a19ddb,37267,1731751077844 2024-11-16T09:59:32,297 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffe1934c280093c181c2b062a05e0908, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T09:59:32,297 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731751162662.ffe1934c280093c181c2b062a05e0908., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 , the old value is region=TestAcidGuarantees,,1731751162662.ffe1934c280093c181c2b062a05e0908., hostname=037c58a19ddb,37267,1731751077844, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffe1934c280093c181c2b062a05e0908, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:59:32,297 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43416 deadline: 1731751232293, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffe1934c280093c181c2b062a05e0908, server=037c58a19ddb,37267,1731751077844 2024-11-16T09:59:32,297 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731751162662.ffe1934c280093c181c2b062a05e0908., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffe1934c280093c181c2b062a05e0908, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:59:32,297 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731751162662.ffe1934c280093c181c2b062a05e0908., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 because the exception is null or not the one we care about 2024-11-16T09:59:32,297 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731751162662.ffe1934c280093c181c2b062a05e0908., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 , the old value is region=TestAcidGuarantees,,1731751162662.ffe1934c280093c181c2b062a05e0908., hostname=037c58a19ddb,37267,1731751077844, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffe1934c280093c181c2b062a05e0908, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:59:32,297 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731751162662.ffe1934c280093c181c2b062a05e0908., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffe1934c280093c181c2b062a05e0908, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:59:32,297 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731751162662.ffe1934c280093c181c2b062a05e0908., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 because the exception is null or not the one we care about 2024-11-16T09:59:32,399 INFO [RS_FLUSH_OPERATIONS-regionserver/037c58a19ddb:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=92 (bloomFilter=true), to=hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/ffe1934c280093c181c2b062a05e0908/.tmp/B/80fb85a1b63c419f8a6bc5905300e5f5 2024-11-16T09:59:32,406 DEBUG [RS_FLUSH_OPERATIONS-regionserver/037c58a19ddb:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/ffe1934c280093c181c2b062a05e0908/.tmp/C/411f65ffbe6f434caf13db4742086cea is 50, key is test_row_0/C:col10/1731751169909/Put/seqid=0 2024-11-16T09:59:32,409 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37591 is added to blk_1073742194_1370 (size=12001) 2024-11-16T09:59:32,509 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffe1934c280093c181c2b062a05e0908, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T09:59:32,510 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43362 deadline: 1731751232502, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffe1934c280093c181c2b062a05e0908, server=037c58a19ddb,37267,1731751077844 2024-11-16T09:59:32,510 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731751162662.ffe1934c280093c181c2b062a05e0908., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 , the old value is region=TestAcidGuarantees,,1731751162662.ffe1934c280093c181c2b062a05e0908., hostname=037c58a19ddb,37267,1731751077844, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffe1934c280093c181c2b062a05e0908, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:59:32,510 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffe1934c280093c181c2b062a05e0908, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T09:59:32,510 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731751162662.ffe1934c280093c181c2b062a05e0908., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffe1934c280093c181c2b062a05e0908, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:59:32,510 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731751162662.ffe1934c280093c181c2b062a05e0908., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 because the exception is null or not the one we care about 2024-11-16T09:59:32,510 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43402 deadline: 1731751232502, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffe1934c280093c181c2b062a05e0908, server=037c58a19ddb,37267,1731751077844 2024-11-16T09:59:32,510 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731751162662.ffe1934c280093c181c2b062a05e0908., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 , the old value is region=TestAcidGuarantees,,1731751162662.ffe1934c280093c181c2b062a05e0908., hostname=037c58a19ddb,37267,1731751077844, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffe1934c280093c181c2b062a05e0908, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:59:32,510 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731751162662.ffe1934c280093c181c2b062a05e0908., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffe1934c280093c181c2b062a05e0908, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:59:32,510 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731751162662.ffe1934c280093c181c2b062a05e0908., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 because the exception is null or not the one we care about 2024-11-16T09:59:32,510 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffe1934c280093c181c2b062a05e0908, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T09:59:32,510 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43418 deadline: 1731751232503, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffe1934c280093c181c2b062a05e0908, server=037c58a19ddb,37267,1731751077844 2024-11-16T09:59:32,511 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731751162662.ffe1934c280093c181c2b062a05e0908., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 , the old value is region=TestAcidGuarantees,,1731751162662.ffe1934c280093c181c2b062a05e0908., hostname=037c58a19ddb,37267,1731751077844, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffe1934c280093c181c2b062a05e0908, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:59:32,511 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731751162662.ffe1934c280093c181c2b062a05e0908., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffe1934c280093c181c2b062a05e0908, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:59:32,511 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffe1934c280093c181c2b062a05e0908, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T09:59:32,511 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731751162662.ffe1934c280093c181c2b062a05e0908., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 because the exception is null or not the one we care about 2024-11-16T09:59:32,511 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43416 deadline: 1731751232503, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffe1934c280093c181c2b062a05e0908, server=037c58a19ddb,37267,1731751077844 2024-11-16T09:59:32,511 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731751162662.ffe1934c280093c181c2b062a05e0908., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 , the old value is region=TestAcidGuarantees,,1731751162662.ffe1934c280093c181c2b062a05e0908., hostname=037c58a19ddb,37267,1731751077844, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffe1934c280093c181c2b062a05e0908, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:59:32,511 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731751162662.ffe1934c280093c181c2b062a05e0908., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffe1934c280093c181c2b062a05e0908, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:59:32,511 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731751162662.ffe1934c280093c181c2b062a05e0908., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 because the exception is null or not the one we care about 2024-11-16T09:59:32,511 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffe1934c280093c181c2b062a05e0908, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T09:59:32,511 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43332 deadline: 1731751232503, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffe1934c280093c181c2b062a05e0908, server=037c58a19ddb,37267,1731751077844 2024-11-16T09:59:32,511 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731751162662.ffe1934c280093c181c2b062a05e0908., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 , the old value is region=TestAcidGuarantees,,1731751162662.ffe1934c280093c181c2b062a05e0908., hostname=037c58a19ddb,37267,1731751077844, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffe1934c280093c181c2b062a05e0908, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:59:32,511 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731751162662.ffe1934c280093c181c2b062a05e0908., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffe1934c280093c181c2b062a05e0908, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:59:32,511 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731751162662.ffe1934c280093c181c2b062a05e0908., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 because the exception is null or not the one we care about 2024-11-16T09:59:32,810 INFO [RS_FLUSH_OPERATIONS-regionserver/037c58a19ddb:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=92 (bloomFilter=true), to=hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/ffe1934c280093c181c2b062a05e0908/.tmp/C/411f65ffbe6f434caf13db4742086cea 2024-11-16T09:59:32,814 DEBUG [RS_FLUSH_OPERATIONS-regionserver/037c58a19ddb:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/ffe1934c280093c181c2b062a05e0908/.tmp/A/70b600411963462196c33bf15fa80c47 as hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/ffe1934c280093c181c2b062a05e0908/A/70b600411963462196c33bf15fa80c47 2024-11-16T09:59:32,818 INFO [RS_FLUSH_OPERATIONS-regionserver/037c58a19ddb:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/ffe1934c280093c181c2b062a05e0908/A/70b600411963462196c33bf15fa80c47, entries=150, sequenceid=92, filesize=30.2 K 2024-11-16T09:59:32,819 DEBUG [RS_FLUSH_OPERATIONS-regionserver/037c58a19ddb:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/ffe1934c280093c181c2b062a05e0908/.tmp/B/80fb85a1b63c419f8a6bc5905300e5f5 as hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/ffe1934c280093c181c2b062a05e0908/B/80fb85a1b63c419f8a6bc5905300e5f5 2024-11-16T09:59:32,822 INFO [RS_FLUSH_OPERATIONS-regionserver/037c58a19ddb:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/ffe1934c280093c181c2b062a05e0908/B/80fb85a1b63c419f8a6bc5905300e5f5, entries=150, sequenceid=92, filesize=11.7 K 2024-11-16T09:59:32,823 DEBUG [RS_FLUSH_OPERATIONS-regionserver/037c58a19ddb:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/ffe1934c280093c181c2b062a05e0908/.tmp/C/411f65ffbe6f434caf13db4742086cea as hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/ffe1934c280093c181c2b062a05e0908/C/411f65ffbe6f434caf13db4742086cea 2024-11-16T09:59:32,827 INFO [RS_FLUSH_OPERATIONS-regionserver/037c58a19ddb:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/ffe1934c280093c181c2b062a05e0908/C/411f65ffbe6f434caf13db4742086cea, entries=150, sequenceid=92, filesize=11.7 K 2024-11-16T09:59:32,827 INFO [RS_FLUSH_OPERATIONS-regionserver/037c58a19ddb:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~158.91 KB/162720, currentSize=147.60 KB/151140 for ffe1934c280093c181c2b062a05e0908 in 1664ms, sequenceid=92, compaction requested=true 2024-11-16T09:59:32,827 DEBUG [RS_FLUSH_OPERATIONS-regionserver/037c58a19ddb:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HRegion(2603): Flush status journal for ffe1934c280093c181c2b062a05e0908: 2024-11-16T09:59:32,828 DEBUG [RS_FLUSH_OPERATIONS-regionserver/037c58a19ddb:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731751162662.ffe1934c280093c181c2b062a05e0908. 2024-11-16T09:59:32,828 DEBUG [RS_FLUSH_OPERATIONS-regionserver/037c58a19ddb:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=100 2024-11-16T09:59:32,828 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37877 {}] master.HMaster(4169): Remote procedure done, pid=100 2024-11-16T09:59:32,830 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] regionserver.HRegion(8855): Flush requested on ffe1934c280093c181c2b062a05e0908 2024-11-16T09:59:32,830 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-16T09:59:32,830 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing ffe1934c280093c181c2b062a05e0908 3/3 column families, dataSize=154.31 KB heapSize=405.05 KB 2024-11-16T09:59:32,831 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=100, resume processing ppid=99 2024-11-16T09:59:32,831 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=100, ppid=99, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.8930 sec 2024-11-16T09:59:32,834 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=99, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=99, table=TestAcidGuarantees in 2.8980 sec 2024-11-16T09:59:32,839 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411160b7c32d8d96e45b0812750bb24a9975c_ffe1934c280093c181c2b062a05e0908 is 50, key is test_row_0/A:col10/1731751172829/Put/seqid=0 2024-11-16T09:59:32,842 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37591 is added to blk_1073742195_1371 (size=17034) 2024-11-16T09:59:32,847 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffe1934c280093c181c2b062a05e0908, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T09:59:32,847 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43362 deadline: 1731751232837, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffe1934c280093c181c2b062a05e0908, server=037c58a19ddb,37267,1731751077844 2024-11-16T09:59:32,848 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffe1934c280093c181c2b062a05e0908, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T09:59:32,848 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731751162662.ffe1934c280093c181c2b062a05e0908., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 , the old value is region=TestAcidGuarantees,,1731751162662.ffe1934c280093c181c2b062a05e0908., hostname=037c58a19ddb,37267,1731751077844, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffe1934c280093c181c2b062a05e0908, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:59:32,848 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731751162662.ffe1934c280093c181c2b062a05e0908., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffe1934c280093c181c2b062a05e0908, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:59:32,848 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731751162662.ffe1934c280093c181c2b062a05e0908., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 because the exception is null or not the one we care about 2024-11-16T09:59:32,848 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43402 deadline: 1731751232846, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffe1934c280093c181c2b062a05e0908, server=037c58a19ddb,37267,1731751077844 2024-11-16T09:59:32,848 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731751162662.ffe1934c280093c181c2b062a05e0908., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 , the old value is region=TestAcidGuarantees,,1731751162662.ffe1934c280093c181c2b062a05e0908., hostname=037c58a19ddb,37267,1731751077844, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffe1934c280093c181c2b062a05e0908, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:59:32,848 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731751162662.ffe1934c280093c181c2b062a05e0908., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffe1934c280093c181c2b062a05e0908, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:59:32,848 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731751162662.ffe1934c280093c181c2b062a05e0908., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 because the exception is null or not the one we care about 2024-11-16T09:59:32,857 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffe1934c280093c181c2b062a05e0908, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T09:59:32,857 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43332 deadline: 1731751232847, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffe1934c280093c181c2b062a05e0908, server=037c58a19ddb,37267,1731751077844 2024-11-16T09:59:32,858 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffe1934c280093c181c2b062a05e0908, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T09:59:32,858 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731751162662.ffe1934c280093c181c2b062a05e0908., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 , the old value is region=TestAcidGuarantees,,1731751162662.ffe1934c280093c181c2b062a05e0908., hostname=037c58a19ddb,37267,1731751077844, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffe1934c280093c181c2b062a05e0908, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:59:32,858 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731751162662.ffe1934c280093c181c2b062a05e0908., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffe1934c280093c181c2b062a05e0908, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:59:32,858 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffe1934c280093c181c2b062a05e0908, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T09:59:32,858 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731751162662.ffe1934c280093c181c2b062a05e0908., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 because the exception is null or not the one we care about 2024-11-16T09:59:32,858 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43416 deadline: 1731751232848, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffe1934c280093c181c2b062a05e0908, server=037c58a19ddb,37267,1731751077844 2024-11-16T09:59:32,858 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43418 deadline: 1731751232848, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffe1934c280093c181c2b062a05e0908, server=037c58a19ddb,37267,1731751077844 2024-11-16T09:59:32,858 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731751162662.ffe1934c280093c181c2b062a05e0908., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 , the old value is region=TestAcidGuarantees,,1731751162662.ffe1934c280093c181c2b062a05e0908., hostname=037c58a19ddb,37267,1731751077844, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffe1934c280093c181c2b062a05e0908, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:59:32,858 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731751162662.ffe1934c280093c181c2b062a05e0908., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffe1934c280093c181c2b062a05e0908, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:59:32,858 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731751162662.ffe1934c280093c181c2b062a05e0908., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 because the exception is null or not the one we care about 2024-11-16T09:59:32,858 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731751162662.ffe1934c280093c181c2b062a05e0908., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 , the old value is region=TestAcidGuarantees,,1731751162662.ffe1934c280093c181c2b062a05e0908., hostname=037c58a19ddb,37267,1731751077844, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffe1934c280093c181c2b062a05e0908, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:59:32,858 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731751162662.ffe1934c280093c181c2b062a05e0908., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffe1934c280093c181c2b062a05e0908, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:59:32,858 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731751162662.ffe1934c280093c181c2b062a05e0908., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 because the exception is null or not the one we care about 2024-11-16T09:59:32,961 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffe1934c280093c181c2b062a05e0908, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T09:59:32,961 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43362 deadline: 1731751232953, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffe1934c280093c181c2b062a05e0908, server=037c58a19ddb,37267,1731751077844 2024-11-16T09:59:32,961 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffe1934c280093c181c2b062a05e0908, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T09:59:32,961 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731751162662.ffe1934c280093c181c2b062a05e0908., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 , the old value is region=TestAcidGuarantees,,1731751162662.ffe1934c280093c181c2b062a05e0908., hostname=037c58a19ddb,37267,1731751077844, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffe1934c280093c181c2b062a05e0908, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:59:32,961 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43402 deadline: 1731751232953, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffe1934c280093c181c2b062a05e0908, server=037c58a19ddb,37267,1731751077844 2024-11-16T09:59:32,961 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731751162662.ffe1934c280093c181c2b062a05e0908., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffe1934c280093c181c2b062a05e0908, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:59:32,961 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731751162662.ffe1934c280093c181c2b062a05e0908., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 because the exception is null or not the one we care about 2024-11-16T09:59:32,962 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731751162662.ffe1934c280093c181c2b062a05e0908., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 , the old value is region=TestAcidGuarantees,,1731751162662.ffe1934c280093c181c2b062a05e0908., hostname=037c58a19ddb,37267,1731751077844, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffe1934c280093c181c2b062a05e0908, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:59:32,962 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731751162662.ffe1934c280093c181c2b062a05e0908., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffe1934c280093c181c2b062a05e0908, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:59:32,962 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731751162662.ffe1934c280093c181c2b062a05e0908., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 because the exception is null or not the one we care about 2024-11-16T09:59:32,964 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffe1934c280093c181c2b062a05e0908, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T09:59:32,964 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43332 deadline: 1731751232963, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffe1934c280093c181c2b062a05e0908, server=037c58a19ddb,37267,1731751077844 2024-11-16T09:59:32,964 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffe1934c280093c181c2b062a05e0908, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T09:59:32,964 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43416 deadline: 1731751232963, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffe1934c280093c181c2b062a05e0908, server=037c58a19ddb,37267,1731751077844 2024-11-16T09:59:32,964 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731751162662.ffe1934c280093c181c2b062a05e0908., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 , the old value is region=TestAcidGuarantees,,1731751162662.ffe1934c280093c181c2b062a05e0908., hostname=037c58a19ddb,37267,1731751077844, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffe1934c280093c181c2b062a05e0908, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:59:32,965 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731751162662.ffe1934c280093c181c2b062a05e0908., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffe1934c280093c181c2b062a05e0908, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:59:32,965 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731751162662.ffe1934c280093c181c2b062a05e0908., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 because the exception is null or not the one we care about 2024-11-16T09:59:32,965 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731751162662.ffe1934c280093c181c2b062a05e0908., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 , the old value is region=TestAcidGuarantees,,1731751162662.ffe1934c280093c181c2b062a05e0908., hostname=037c58a19ddb,37267,1731751077844, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffe1934c280093c181c2b062a05e0908, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:59:32,965 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731751162662.ffe1934c280093c181c2b062a05e0908., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffe1934c280093c181c2b062a05e0908, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:59:32,965 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731751162662.ffe1934c280093c181c2b062a05e0908., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 because the exception is null or not the one we care about 2024-11-16T09:59:32,973 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffe1934c280093c181c2b062a05e0908, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T09:59:32,973 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43418 deadline: 1731751232963, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffe1934c280093c181c2b062a05e0908, server=037c58a19ddb,37267,1731751077844 2024-11-16T09:59:32,973 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731751162662.ffe1934c280093c181c2b062a05e0908., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 , the old value is region=TestAcidGuarantees,,1731751162662.ffe1934c280093c181c2b062a05e0908., hostname=037c58a19ddb,37267,1731751077844, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffe1934c280093c181c2b062a05e0908, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:59:32,973 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731751162662.ffe1934c280093c181c2b062a05e0908., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffe1934c280093c181c2b062a05e0908, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:59:32,973 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731751162662.ffe1934c280093c181c2b062a05e0908., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 because the exception is null or not the one we care about 2024-11-16T09:59:33,182 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffe1934c280093c181c2b062a05e0908, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T09:59:33,182 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43362 deadline: 1731751233173, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffe1934c280093c181c2b062a05e0908, server=037c58a19ddb,37267,1731751077844 2024-11-16T09:59:33,182 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffe1934c280093c181c2b062a05e0908, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T09:59:33,182 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731751162662.ffe1934c280093c181c2b062a05e0908., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 , the old value is region=TestAcidGuarantees,,1731751162662.ffe1934c280093c181c2b062a05e0908., hostname=037c58a19ddb,37267,1731751077844, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffe1934c280093c181c2b062a05e0908, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:59:33,183 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731751162662.ffe1934c280093c181c2b062a05e0908., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffe1934c280093c181c2b062a05e0908, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:59:33,183 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43402 deadline: 1731751233173, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffe1934c280093c181c2b062a05e0908, server=037c58a19ddb,37267,1731751077844 2024-11-16T09:59:33,183 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731751162662.ffe1934c280093c181c2b062a05e0908., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 because the exception is null or not the one we care about 2024-11-16T09:59:33,183 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffe1934c280093c181c2b062a05e0908, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T09:59:33,183 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731751162662.ffe1934c280093c181c2b062a05e0908., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 , the old value is region=TestAcidGuarantees,,1731751162662.ffe1934c280093c181c2b062a05e0908., hostname=037c58a19ddb,37267,1731751077844, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffe1934c280093c181c2b062a05e0908, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:59:33,183 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43332 deadline: 1731751233173, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffe1934c280093c181c2b062a05e0908, server=037c58a19ddb,37267,1731751077844 2024-11-16T09:59:33,183 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731751162662.ffe1934c280093c181c2b062a05e0908., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffe1934c280093c181c2b062a05e0908, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:59:33,183 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731751162662.ffe1934c280093c181c2b062a05e0908., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 because the exception is null or not the one we care about 2024-11-16T09:59:33,183 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffe1934c280093c181c2b062a05e0908, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T09:59:33,183 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43416 deadline: 1731751233173, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffe1934c280093c181c2b062a05e0908, server=037c58a19ddb,37267,1731751077844 2024-11-16T09:59:33,183 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731751162662.ffe1934c280093c181c2b062a05e0908., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 , the old value is region=TestAcidGuarantees,,1731751162662.ffe1934c280093c181c2b062a05e0908., hostname=037c58a19ddb,37267,1731751077844, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffe1934c280093c181c2b062a05e0908, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:59:33,183 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731751162662.ffe1934c280093c181c2b062a05e0908., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffe1934c280093c181c2b062a05e0908, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:59:33,183 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731751162662.ffe1934c280093c181c2b062a05e0908., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 because the exception is null or not the one we care about 2024-11-16T09:59:33,183 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731751162662.ffe1934c280093c181c2b062a05e0908., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 , the old value is region=TestAcidGuarantees,,1731751162662.ffe1934c280093c181c2b062a05e0908., hostname=037c58a19ddb,37267,1731751077844, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffe1934c280093c181c2b062a05e0908, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:59:33,183 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731751162662.ffe1934c280093c181c2b062a05e0908., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffe1934c280093c181c2b062a05e0908, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:59:33,183 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731751162662.ffe1934c280093c181c2b062a05e0908., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 because the exception is null or not the one we care about 2024-11-16T09:59:33,187 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffe1934c280093c181c2b062a05e0908, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T09:59:33,187 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43418 deadline: 1731751233182, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffe1934c280093c181c2b062a05e0908, server=037c58a19ddb,37267,1731751077844 2024-11-16T09:59:33,187 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731751162662.ffe1934c280093c181c2b062a05e0908., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 , the old value is region=TestAcidGuarantees,,1731751162662.ffe1934c280093c181c2b062a05e0908., hostname=037c58a19ddb,37267,1731751077844, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffe1934c280093c181c2b062a05e0908, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:59:33,187 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731751162662.ffe1934c280093c181c2b062a05e0908., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffe1934c280093c181c2b062a05e0908, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:59:33,187 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731751162662.ffe1934c280093c181c2b062a05e0908., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 because the exception is null or not the one we care about 2024-11-16T09:59:33,243 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T09:59:33,246 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411160b7c32d8d96e45b0812750bb24a9975c_ffe1934c280093c181c2b062a05e0908 to hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411160b7c32d8d96e45b0812750bb24a9975c_ffe1934c280093c181c2b062a05e0908 2024-11-16T09:59:33,247 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/ffe1934c280093c181c2b062a05e0908/.tmp/A/f9a4ab02427a4739bf321279a7bcb212, store: [table=TestAcidGuarantees family=A region=ffe1934c280093c181c2b062a05e0908] 2024-11-16T09:59:33,248 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/ffe1934c280093c181c2b062a05e0908/.tmp/A/f9a4ab02427a4739bf321279a7bcb212 is 175, key is test_row_0/A:col10/1731751172829/Put/seqid=0 2024-11-16T09:59:33,252 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37591 is added to blk_1073742196_1372 (size=48139) 2024-11-16T09:59:33,498 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffe1934c280093c181c2b062a05e0908, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T09:59:33,498 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43362 deadline: 1731751233493, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffe1934c280093c181c2b062a05e0908, server=037c58a19ddb,37267,1731751077844 2024-11-16T09:59:33,498 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffe1934c280093c181c2b062a05e0908, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T09:59:33,499 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731751162662.ffe1934c280093c181c2b062a05e0908., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 , the old value is region=TestAcidGuarantees,,1731751162662.ffe1934c280093c181c2b062a05e0908., hostname=037c58a19ddb,37267,1731751077844, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffe1934c280093c181c2b062a05e0908, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:59:33,499 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43402 deadline: 1731751233493, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffe1934c280093c181c2b062a05e0908, server=037c58a19ddb,37267,1731751077844 2024-11-16T09:59:33,499 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731751162662.ffe1934c280093c181c2b062a05e0908., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffe1934c280093c181c2b062a05e0908, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:59:33,499 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731751162662.ffe1934c280093c181c2b062a05e0908., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 because the exception is null or not the one we care about 2024-11-16T09:59:33,499 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731751162662.ffe1934c280093c181c2b062a05e0908., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 , the old value is region=TestAcidGuarantees,,1731751162662.ffe1934c280093c181c2b062a05e0908., hostname=037c58a19ddb,37267,1731751077844, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffe1934c280093c181c2b062a05e0908, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:59:33,499 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731751162662.ffe1934c280093c181c2b062a05e0908., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffe1934c280093c181c2b062a05e0908, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:59:33,499 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffe1934c280093c181c2b062a05e0908, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T09:59:33,499 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731751162662.ffe1934c280093c181c2b062a05e0908., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 because the exception is null or not the one we care about 2024-11-16T09:59:33,499 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43332 deadline: 1731751233493, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffe1934c280093c181c2b062a05e0908, server=037c58a19ddb,37267,1731751077844 2024-11-16T09:59:33,499 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731751162662.ffe1934c280093c181c2b062a05e0908., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 , the old value is region=TestAcidGuarantees,,1731751162662.ffe1934c280093c181c2b062a05e0908., hostname=037c58a19ddb,37267,1731751077844, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffe1934c280093c181c2b062a05e0908, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:59:33,499 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffe1934c280093c181c2b062a05e0908, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T09:59:33,499 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731751162662.ffe1934c280093c181c2b062a05e0908., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffe1934c280093c181c2b062a05e0908, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:59:33,499 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731751162662.ffe1934c280093c181c2b062a05e0908., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 because the exception is null or not the one we care about 2024-11-16T09:59:33,499 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43416 deadline: 1731751233493, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffe1934c280093c181c2b062a05e0908, server=037c58a19ddb,37267,1731751077844 2024-11-16T09:59:33,499 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731751162662.ffe1934c280093c181c2b062a05e0908., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 , the old value is region=TestAcidGuarantees,,1731751162662.ffe1934c280093c181c2b062a05e0908., hostname=037c58a19ddb,37267,1731751077844, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffe1934c280093c181c2b062a05e0908, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:59:33,499 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731751162662.ffe1934c280093c181c2b062a05e0908., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffe1934c280093c181c2b062a05e0908, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:59:33,499 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffe1934c280093c181c2b062a05e0908, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T09:59:33,500 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731751162662.ffe1934c280093c181c2b062a05e0908., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 because the exception is null or not the one we care about 2024-11-16T09:59:33,500 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43418 deadline: 1731751233493, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffe1934c280093c181c2b062a05e0908, server=037c58a19ddb,37267,1731751077844 2024-11-16T09:59:33,500 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731751162662.ffe1934c280093c181c2b062a05e0908., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 , the old value is region=TestAcidGuarantees,,1731751162662.ffe1934c280093c181c2b062a05e0908., hostname=037c58a19ddb,37267,1731751077844, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffe1934c280093c181c2b062a05e0908, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:59:33,500 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731751162662.ffe1934c280093c181c2b062a05e0908., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffe1934c280093c181c2b062a05e0908, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:59:33,500 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731751162662.ffe1934c280093c181c2b062a05e0908., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 because the exception is null or not the one we care about 2024-11-16T09:59:33,653 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=119, memsize=53.7 K, hasBloomFilter=true, into tmp file hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/ffe1934c280093c181c2b062a05e0908/.tmp/A/f9a4ab02427a4739bf321279a7bcb212 2024-11-16T09:59:33,661 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/ffe1934c280093c181c2b062a05e0908/.tmp/B/454a223dd32245fda138a60a07d8ee12 is 50, key is test_row_0/B:col10/1731751172829/Put/seqid=0 2024-11-16T09:59:33,665 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37591 is added to blk_1073742197_1373 (size=12001) 2024-11-16T09:59:34,012 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffe1934c280093c181c2b062a05e0908, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T09:59:34,013 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43362 deadline: 1731751234003, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffe1934c280093c181c2b062a05e0908, server=037c58a19ddb,37267,1731751077844 2024-11-16T09:59:34,013 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffe1934c280093c181c2b062a05e0908, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T09:59:34,013 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731751162662.ffe1934c280093c181c2b062a05e0908., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 , the old value is region=TestAcidGuarantees,,1731751162662.ffe1934c280093c181c2b062a05e0908., hostname=037c58a19ddb,37267,1731751077844, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffe1934c280093c181c2b062a05e0908, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:59:34,013 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43402 deadline: 1731751234003, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffe1934c280093c181c2b062a05e0908, server=037c58a19ddb,37267,1731751077844 2024-11-16T09:59:34,013 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731751162662.ffe1934c280093c181c2b062a05e0908., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffe1934c280093c181c2b062a05e0908, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:59:34,013 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731751162662.ffe1934c280093c181c2b062a05e0908., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 because the exception is null or not the one we care about 2024-11-16T09:59:34,013 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731751162662.ffe1934c280093c181c2b062a05e0908., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 , the old value is region=TestAcidGuarantees,,1731751162662.ffe1934c280093c181c2b062a05e0908., hostname=037c58a19ddb,37267,1731751077844, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffe1934c280093c181c2b062a05e0908, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:59:34,013 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffe1934c280093c181c2b062a05e0908, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T09:59:34,013 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731751162662.ffe1934c280093c181c2b062a05e0908., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffe1934c280093c181c2b062a05e0908, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:59:34,013 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731751162662.ffe1934c280093c181c2b062a05e0908., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 because the exception is null or not the one we care about 2024-11-16T09:59:34,013 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43416 deadline: 1731751234003, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffe1934c280093c181c2b062a05e0908, server=037c58a19ddb,37267,1731751077844 2024-11-16T09:59:34,014 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731751162662.ffe1934c280093c181c2b062a05e0908., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 , the old value is region=TestAcidGuarantees,,1731751162662.ffe1934c280093c181c2b062a05e0908., hostname=037c58a19ddb,37267,1731751077844, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffe1934c280093c181c2b062a05e0908, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:59:34,014 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731751162662.ffe1934c280093c181c2b062a05e0908., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffe1934c280093c181c2b062a05e0908, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:59:34,014 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731751162662.ffe1934c280093c181c2b062a05e0908., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 because the exception is null or not the one we care about 2024-11-16T09:59:34,020 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffe1934c280093c181c2b062a05e0908, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T09:59:34,020 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43332 deadline: 1731751234012, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffe1934c280093c181c2b062a05e0908, server=037c58a19ddb,37267,1731751077844 2024-11-16T09:59:34,020 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731751162662.ffe1934c280093c181c2b062a05e0908., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 , the old value is region=TestAcidGuarantees,,1731751162662.ffe1934c280093c181c2b062a05e0908., hostname=037c58a19ddb,37267,1731751077844, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffe1934c280093c181c2b062a05e0908, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:59:34,020 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffe1934c280093c181c2b062a05e0908, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T09:59:34,020 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731751162662.ffe1934c280093c181c2b062a05e0908., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffe1934c280093c181c2b062a05e0908, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:59:34,020 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731751162662.ffe1934c280093c181c2b062a05e0908., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 because the exception is null or not the one we care about 2024-11-16T09:59:34,020 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43418 deadline: 1731751234012, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffe1934c280093c181c2b062a05e0908, server=037c58a19ddb,37267,1731751077844 2024-11-16T09:59:34,020 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731751162662.ffe1934c280093c181c2b062a05e0908., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 , the old value is region=TestAcidGuarantees,,1731751162662.ffe1934c280093c181c2b062a05e0908., hostname=037c58a19ddb,37267,1731751077844, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffe1934c280093c181c2b062a05e0908, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:59:34,020 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731751162662.ffe1934c280093c181c2b062a05e0908., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffe1934c280093c181c2b062a05e0908, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:59:34,020 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731751162662.ffe1934c280093c181c2b062a05e0908., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 because the exception is null or not the one we care about 2024-11-16T09:59:34,065 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=53.67 KB at sequenceid=119 (bloomFilter=true), to=hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/ffe1934c280093c181c2b062a05e0908/.tmp/B/454a223dd32245fda138a60a07d8ee12 2024-11-16T09:59:34,073 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/ffe1934c280093c181c2b062a05e0908/.tmp/C/c5a878457eff483bb100e1b51d494e7f is 50, key is test_row_0/C:col10/1731751172829/Put/seqid=0 2024-11-16T09:59:34,077 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37591 is added to blk_1073742198_1374 (size=12001) 2024-11-16T09:59:34,103 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37877 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=99 2024-11-16T09:59:34,104 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-16T09:59:34,104 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37877 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-16T09:59:34,105 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37877 {}] procedure2.ProcedureExecutor(1139): Stored pid=101, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=101, table=TestAcidGuarantees 2024-11-16T09:59:34,106 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37877 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=101 2024-11-16T09:59:34,106 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=101, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=101, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-16T09:59:34,107 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=101, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=101, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-16T09:59:34,107 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=102, ppid=101, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-16T09:59:34,213 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37877 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=101 2024-11-16T09:59:34,264 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=37267 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=102 2024-11-16T09:59:34,264 DEBUG [RS_FLUSH_OPERATIONS-regionserver/037c58a19ddb:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731751162662.ffe1934c280093c181c2b062a05e0908. 2024-11-16T09:59:34,264 DEBUG [RS_FLUSH_OPERATIONS-regionserver/037c58a19ddb:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731751162662.ffe1934c280093c181c2b062a05e0908. as already flushing 2024-11-16T09:59:34,264 DEBUG [RS_FLUSH_OPERATIONS-regionserver/037c58a19ddb:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731751162662.ffe1934c280093c181c2b062a05e0908. 2024-11-16T09:59:34,264 ERROR [RS_FLUSH_OPERATIONS-regionserver/037c58a19ddb:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] handler.RSProcedureHandler(58): pid=102 java.io.IOException: Unable to complete flush {ENCODED => ffe1934c280093c181c2b062a05e0908, NAME => 'TestAcidGuarantees,,1731751162662.ffe1934c280093c181c2b062a05e0908.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T09:59:34,264 DEBUG [RS_FLUSH_OPERATIONS-regionserver/037c58a19ddb:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=102 java.io.IOException: Unable to complete flush {ENCODED => ffe1934c280093c181c2b062a05e0908, NAME => 'TestAcidGuarantees,,1731751162662.ffe1934c280093c181c2b062a05e0908.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T09:59:34,265 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37877 {}] master.HMaster(4177): Remote procedure failed, pid=102 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ffe1934c280093c181c2b062a05e0908, NAME => 'TestAcidGuarantees,,1731751162662.ffe1934c280093c181c2b062a05e0908.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ffe1934c280093c181c2b062a05e0908, NAME => 'TestAcidGuarantees,,1731751162662.ffe1934c280093c181c2b062a05e0908.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T09:59:34,417 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=37267 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=102 2024-11-16T09:59:34,417 DEBUG [RS_FLUSH_OPERATIONS-regionserver/037c58a19ddb:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731751162662.ffe1934c280093c181c2b062a05e0908. 2024-11-16T09:59:34,417 DEBUG [RS_FLUSH_OPERATIONS-regionserver/037c58a19ddb:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731751162662.ffe1934c280093c181c2b062a05e0908. as already flushing 2024-11-16T09:59:34,417 DEBUG [RS_FLUSH_OPERATIONS-regionserver/037c58a19ddb:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731751162662.ffe1934c280093c181c2b062a05e0908. 2024-11-16T09:59:34,418 ERROR [RS_FLUSH_OPERATIONS-regionserver/037c58a19ddb:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] handler.RSProcedureHandler(58): pid=102 java.io.IOException: Unable to complete flush {ENCODED => ffe1934c280093c181c2b062a05e0908, NAME => 'TestAcidGuarantees,,1731751162662.ffe1934c280093c181c2b062a05e0908.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T09:59:34,418 DEBUG [RS_FLUSH_OPERATIONS-regionserver/037c58a19ddb:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=102 java.io.IOException: Unable to complete flush {ENCODED => ffe1934c280093c181c2b062a05e0908, NAME => 'TestAcidGuarantees,,1731751162662.ffe1934c280093c181c2b062a05e0908.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T09:59:34,418 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37877 {}] master.HMaster(4177): Remote procedure failed, pid=102 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ffe1934c280093c181c2b062a05e0908, NAME => 'TestAcidGuarantees,,1731751162662.ffe1934c280093c181c2b062a05e0908.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ffe1934c280093c181c2b062a05e0908, NAME => 'TestAcidGuarantees,,1731751162662.ffe1934c280093c181c2b062a05e0908.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T09:59:34,423 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37877 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=101 2024-11-16T09:59:34,478 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=53.67 KB at sequenceid=119 (bloomFilter=true), to=hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/ffe1934c280093c181c2b062a05e0908/.tmp/C/c5a878457eff483bb100e1b51d494e7f 2024-11-16T09:59:34,482 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/ffe1934c280093c181c2b062a05e0908/.tmp/A/f9a4ab02427a4739bf321279a7bcb212 as hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/ffe1934c280093c181c2b062a05e0908/A/f9a4ab02427a4739bf321279a7bcb212 2024-11-16T09:59:34,486 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/ffe1934c280093c181c2b062a05e0908/A/f9a4ab02427a4739bf321279a7bcb212, entries=250, sequenceid=119, filesize=47.0 K 2024-11-16T09:59:34,487 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/ffe1934c280093c181c2b062a05e0908/.tmp/B/454a223dd32245fda138a60a07d8ee12 as hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/ffe1934c280093c181c2b062a05e0908/B/454a223dd32245fda138a60a07d8ee12 2024-11-16T09:59:34,490 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/ffe1934c280093c181c2b062a05e0908/B/454a223dd32245fda138a60a07d8ee12, entries=150, sequenceid=119, filesize=11.7 K 2024-11-16T09:59:34,491 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/ffe1934c280093c181c2b062a05e0908/.tmp/C/c5a878457eff483bb100e1b51d494e7f as hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/ffe1934c280093c181c2b062a05e0908/C/c5a878457eff483bb100e1b51d494e7f 2024-11-16T09:59:34,494 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/ffe1934c280093c181c2b062a05e0908/C/c5a878457eff483bb100e1b51d494e7f, entries=150, sequenceid=119, filesize=11.7 K 2024-11-16T09:59:34,495 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~161.02 KB/164880, heapSize ~422.58 KB/432720, currentSize=53.67 KB/54960 for ffe1934c280093c181c2b062a05e0908 in 1665ms, sequenceid=119, compaction requested=true 2024-11-16T09:59:34,495 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for ffe1934c280093c181c2b062a05e0908: 2024-11-16T09:59:34,496 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store ffe1934c280093c181c2b062a05e0908:A, priority=-2147483648, current under compaction store size is 1 2024-11-16T09:59:34,496 DEBUG [RS:0;037c58a19ddb:37267-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-16T09:59:34,496 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-16T09:59:34,496 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store ffe1934c280093c181c2b062a05e0908:B, priority=-2147483648, current under compaction store size is 2 2024-11-16T09:59:34,496 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-16T09:59:34,496 DEBUG [RS:0;037c58a19ddb:37267-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-16T09:59:34,496 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store ffe1934c280093c181c2b062a05e0908:C, priority=-2147483648, current under compaction store size is 3 2024-11-16T09:59:34,496 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-16T09:59:34,497 DEBUG [RS:0;037c58a19ddb:37267-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48107 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-16T09:59:34,497 DEBUG [RS:0;037c58a19ddb:37267-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 149701 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-16T09:59:34,497 DEBUG [RS:0;037c58a19ddb:37267-longCompactions-0 {}] regionserver.HStore(1541): ffe1934c280093c181c2b062a05e0908/B is initiating minor compaction (all files) 2024-11-16T09:59:34,497 DEBUG [RS:0;037c58a19ddb:37267-shortCompactions-0 {}] regionserver.HStore(1541): ffe1934c280093c181c2b062a05e0908/A is initiating minor compaction (all files) 2024-11-16T09:59:34,497 INFO [RS:0;037c58a19ddb:37267-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of ffe1934c280093c181c2b062a05e0908/A in TestAcidGuarantees,,1731751162662.ffe1934c280093c181c2b062a05e0908. 2024-11-16T09:59:34,497 INFO [RS:0;037c58a19ddb:37267-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of ffe1934c280093c181c2b062a05e0908/B in TestAcidGuarantees,,1731751162662.ffe1934c280093c181c2b062a05e0908. 2024-11-16T09:59:34,497 INFO [RS:0;037c58a19ddb:37267-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/ffe1934c280093c181c2b062a05e0908/B/be4ceea2e8b74c3fb5af6601e8cc803b, hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/ffe1934c280093c181c2b062a05e0908/B/d6f737504c674d9397fba019aa8885a5, hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/ffe1934c280093c181c2b062a05e0908/B/80fb85a1b63c419f8a6bc5905300e5f5, hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/ffe1934c280093c181c2b062a05e0908/B/454a223dd32245fda138a60a07d8ee12] into tmpdir=hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/ffe1934c280093c181c2b062a05e0908/.tmp, totalSize=47.0 K 2024-11-16T09:59:34,497 INFO [RS:0;037c58a19ddb:37267-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/ffe1934c280093c181c2b062a05e0908/A/4939aa1e225f4585a4c8405038aea298, hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/ffe1934c280093c181c2b062a05e0908/A/6d3471f357e64c24aa32cc3a11602671, hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/ffe1934c280093c181c2b062a05e0908/A/70b600411963462196c33bf15fa80c47, hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/ffe1934c280093c181c2b062a05e0908/A/f9a4ab02427a4739bf321279a7bcb212] into tmpdir=hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/ffe1934c280093c181c2b062a05e0908/.tmp, totalSize=146.2 K 2024-11-16T09:59:34,497 INFO [RS:0;037c58a19ddb:37267-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=12 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731751162662.ffe1934c280093c181c2b062a05e0908. 2024-11-16T09:59:34,497 DEBUG [RS:0;037c58a19ddb:37267-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731751162662.ffe1934c280093c181c2b062a05e0908. files: [hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/ffe1934c280093c181c2b062a05e0908/A/4939aa1e225f4585a4c8405038aea298, hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/ffe1934c280093c181c2b062a05e0908/A/6d3471f357e64c24aa32cc3a11602671, hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/ffe1934c280093c181c2b062a05e0908/A/70b600411963462196c33bf15fa80c47, hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/ffe1934c280093c181c2b062a05e0908/A/f9a4ab02427a4739bf321279a7bcb212] 2024-11-16T09:59:34,497 DEBUG [RS:0;037c58a19ddb:37267-longCompactions-0 {}] compactions.Compactor(225): Compacting be4ceea2e8b74c3fb5af6601e8cc803b, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=53, earliestPutTs=1731751166994 2024-11-16T09:59:34,498 DEBUG [RS:0;037c58a19ddb:37267-shortCompactions-0 {}] compactions.Compactor(225): Compacting 4939aa1e225f4585a4c8405038aea298, keycount=150, bloomtype=ROW, size=30.3 K, encoding=NONE, compression=NONE, seqNum=53, earliestPutTs=1731751166994 2024-11-16T09:59:34,498 DEBUG [RS:0;037c58a19ddb:37267-longCompactions-0 {}] compactions.Compactor(225): Compacting d6f737504c674d9397fba019aa8885a5, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=77, earliestPutTs=1731751169250 2024-11-16T09:59:34,498 DEBUG [RS:0;037c58a19ddb:37267-shortCompactions-0 {}] compactions.Compactor(225): Compacting 6d3471f357e64c24aa32cc3a11602671, keycount=200, bloomtype=ROW, size=38.6 K, encoding=NONE, compression=NONE, seqNum=77, earliestPutTs=1731751169250 2024-11-16T09:59:34,498 DEBUG [RS:0;037c58a19ddb:37267-longCompactions-0 {}] compactions.Compactor(225): Compacting 80fb85a1b63c419f8a6bc5905300e5f5, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=92, earliestPutTs=1731751169899 2024-11-16T09:59:34,498 DEBUG [RS:0;037c58a19ddb:37267-shortCompactions-0 {}] compactions.Compactor(225): Compacting 70b600411963462196c33bf15fa80c47, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=92, earliestPutTs=1731751169899 2024-11-16T09:59:34,498 DEBUG [RS:0;037c58a19ddb:37267-longCompactions-0 {}] compactions.Compactor(225): Compacting 454a223dd32245fda138a60a07d8ee12, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=119, earliestPutTs=1731751172157 2024-11-16T09:59:34,498 DEBUG [RS:0;037c58a19ddb:37267-shortCompactions-0 {}] compactions.Compactor(225): Compacting f9a4ab02427a4739bf321279a7bcb212, keycount=250, bloomtype=ROW, size=47.0 K, encoding=NONE, compression=NONE, seqNum=119, earliestPutTs=1731751172149 2024-11-16T09:59:34,508 INFO [RS:0;037c58a19ddb:37267-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): ffe1934c280093c181c2b062a05e0908#B#compaction#320 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-16T09:59:34,509 DEBUG [RS:0;037c58a19ddb:37267-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/ffe1934c280093c181c2b062a05e0908/.tmp/B/02659887633940278476ed746fc4b512 is 50, key is test_row_0/B:col10/1731751172829/Put/seqid=0 2024-11-16T09:59:34,512 INFO [RS:0;037c58a19ddb:37267-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=ffe1934c280093c181c2b062a05e0908] 2024-11-16T09:59:34,514 DEBUG [RS:0;037c58a19ddb:37267-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e202411165badad801d8148daa9a9c1d9d7280a43_ffe1934c280093c181c2b062a05e0908 store=[table=TestAcidGuarantees family=A region=ffe1934c280093c181c2b062a05e0908] 2024-11-16T09:59:34,516 DEBUG [RS:0;037c58a19ddb:37267-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e202411165badad801d8148daa9a9c1d9d7280a43_ffe1934c280093c181c2b062a05e0908, store=[table=TestAcidGuarantees family=A region=ffe1934c280093c181c2b062a05e0908] 2024-11-16T09:59:34,517 DEBUG [RS:0;037c58a19ddb:37267-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411165badad801d8148daa9a9c1d9d7280a43_ffe1934c280093c181c2b062a05e0908 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=ffe1934c280093c181c2b062a05e0908] 2024-11-16T09:59:34,518 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37591 is added to blk_1073742199_1375 (size=12241) 2024-11-16T09:59:34,524 DEBUG [RS:0;037c58a19ddb:37267-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/ffe1934c280093c181c2b062a05e0908/.tmp/B/02659887633940278476ed746fc4b512 as hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/ffe1934c280093c181c2b062a05e0908/B/02659887633940278476ed746fc4b512 2024-11-16T09:59:34,526 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37591 is added to blk_1073742200_1376 (size=4469) 2024-11-16T09:59:34,527 INFO [RS:0;037c58a19ddb:37267-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): ffe1934c280093c181c2b062a05e0908#A#compaction#321 average throughput is 1.63 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-16T09:59:34,528 DEBUG [RS:0;037c58a19ddb:37267-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/ffe1934c280093c181c2b062a05e0908/.tmp/A/b94361fb6b4448359e6a3cb995534553 is 175, key is test_row_0/A:col10/1731751172829/Put/seqid=0 2024-11-16T09:59:34,529 INFO [RS:0;037c58a19ddb:37267-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in ffe1934c280093c181c2b062a05e0908/B of ffe1934c280093c181c2b062a05e0908 into 02659887633940278476ed746fc4b512(size=12.0 K), total size for store is 12.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-16T09:59:34,529 DEBUG [RS:0;037c58a19ddb:37267-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for ffe1934c280093c181c2b062a05e0908: 2024-11-16T09:59:34,529 INFO [RS:0;037c58a19ddb:37267-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731751162662.ffe1934c280093c181c2b062a05e0908., storeName=ffe1934c280093c181c2b062a05e0908/B, priority=12, startTime=1731751174496; duration=0sec 2024-11-16T09:59:34,529 DEBUG [RS:0;037c58a19ddb:37267-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-16T09:59:34,529 DEBUG [RS:0;037c58a19ddb:37267-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: ffe1934c280093c181c2b062a05e0908:B 2024-11-16T09:59:34,529 DEBUG [RS:0;037c58a19ddb:37267-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-16T09:59:34,531 DEBUG [RS:0;037c58a19ddb:37267-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48107 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-16T09:59:34,531 DEBUG [RS:0;037c58a19ddb:37267-longCompactions-0 {}] regionserver.HStore(1541): ffe1934c280093c181c2b062a05e0908/C is initiating minor compaction (all files) 2024-11-16T09:59:34,531 INFO [RS:0;037c58a19ddb:37267-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of ffe1934c280093c181c2b062a05e0908/C in TestAcidGuarantees,,1731751162662.ffe1934c280093c181c2b062a05e0908. 2024-11-16T09:59:34,531 INFO [RS:0;037c58a19ddb:37267-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/ffe1934c280093c181c2b062a05e0908/C/ca4c37e2417b4a24b1ff87c6e2fd47f9, hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/ffe1934c280093c181c2b062a05e0908/C/42861dc2507c4c0f8d9b4b330a334f9b, hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/ffe1934c280093c181c2b062a05e0908/C/411f65ffbe6f434caf13db4742086cea, hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/ffe1934c280093c181c2b062a05e0908/C/c5a878457eff483bb100e1b51d494e7f] into tmpdir=hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/ffe1934c280093c181c2b062a05e0908/.tmp, totalSize=47.0 K 2024-11-16T09:59:34,532 DEBUG [RS:0;037c58a19ddb:37267-longCompactions-0 {}] compactions.Compactor(225): Compacting ca4c37e2417b4a24b1ff87c6e2fd47f9, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=53, earliestPutTs=1731751166994 2024-11-16T09:59:34,532 DEBUG [RS:0;037c58a19ddb:37267-longCompactions-0 {}] compactions.Compactor(225): Compacting 42861dc2507c4c0f8d9b4b330a334f9b, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=77, earliestPutTs=1731751169250 2024-11-16T09:59:34,532 DEBUG [RS:0;037c58a19ddb:37267-longCompactions-0 {}] compactions.Compactor(225): Compacting 411f65ffbe6f434caf13db4742086cea, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=92, earliestPutTs=1731751169899 2024-11-16T09:59:34,533 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37591 is added to blk_1073742201_1377 (size=31195) 2024-11-16T09:59:34,533 DEBUG [RS:0;037c58a19ddb:37267-longCompactions-0 {}] compactions.Compactor(225): Compacting c5a878457eff483bb100e1b51d494e7f, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=119, earliestPutTs=1731751172157 2024-11-16T09:59:34,539 DEBUG [RS:0;037c58a19ddb:37267-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/ffe1934c280093c181c2b062a05e0908/.tmp/A/b94361fb6b4448359e6a3cb995534553 as hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/ffe1934c280093c181c2b062a05e0908/A/b94361fb6b4448359e6a3cb995534553 2024-11-16T09:59:34,543 INFO [RS:0;037c58a19ddb:37267-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): ffe1934c280093c181c2b062a05e0908#C#compaction#322 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-16T09:59:34,543 DEBUG [RS:0;037c58a19ddb:37267-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/ffe1934c280093c181c2b062a05e0908/.tmp/C/a53a9c0af5e648b2bedbfa0d8cb8a65b is 50, key is test_row_0/C:col10/1731751172829/Put/seqid=0 2024-11-16T09:59:34,544 INFO [RS:0;037c58a19ddb:37267-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in ffe1934c280093c181c2b062a05e0908/A of ffe1934c280093c181c2b062a05e0908 into b94361fb6b4448359e6a3cb995534553(size=30.5 K), total size for store is 30.5 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-16T09:59:34,544 DEBUG [RS:0;037c58a19ddb:37267-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for ffe1934c280093c181c2b062a05e0908: 2024-11-16T09:59:34,544 INFO [RS:0;037c58a19ddb:37267-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731751162662.ffe1934c280093c181c2b062a05e0908., storeName=ffe1934c280093c181c2b062a05e0908/A, priority=12, startTime=1731751174495; duration=0sec 2024-11-16T09:59:34,544 DEBUG [RS:0;037c58a19ddb:37267-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-16T09:59:34,544 DEBUG [RS:0;037c58a19ddb:37267-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: ffe1934c280093c181c2b062a05e0908:A 2024-11-16T09:59:34,550 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37591 is added to blk_1073742202_1378 (size=12241) 2024-11-16T09:59:34,570 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=37267 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=102 2024-11-16T09:59:34,570 DEBUG [RS_FLUSH_OPERATIONS-regionserver/037c58a19ddb:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731751162662.ffe1934c280093c181c2b062a05e0908. 2024-11-16T09:59:34,571 INFO [RS_FLUSH_OPERATIONS-regionserver/037c58a19ddb:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HRegion(2902): Flushing ffe1934c280093c181c2b062a05e0908 3/3 column families, dataSize=53.67 KB heapSize=141.38 KB 2024-11-16T09:59:34,579 DEBUG [RS_FLUSH_OPERATIONS-regionserver/037c58a19ddb:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411165d7175141f574c2c8081aa20a6cc4783_ffe1934c280093c181c2b062a05e0908 is 50, key is test_row_0/A:col10/1731751172846/Put/seqid=0 2024-11-16T09:59:34,582 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37591 is added to blk_1073742203_1379 (size=12204) 2024-11-16T09:59:34,583 DEBUG [RS_FLUSH_OPERATIONS-regionserver/037c58a19ddb:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T09:59:34,586 INFO [RS_FLUSH_OPERATIONS-regionserver/037c58a19ddb:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411165d7175141f574c2c8081aa20a6cc4783_ffe1934c280093c181c2b062a05e0908 to hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411165d7175141f574c2c8081aa20a6cc4783_ffe1934c280093c181c2b062a05e0908 2024-11-16T09:59:34,587 DEBUG [RS_FLUSH_OPERATIONS-regionserver/037c58a19ddb:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/ffe1934c280093c181c2b062a05e0908/.tmp/A/fc3e9b5bbfd24a0480896ecf8db38f28, store: [table=TestAcidGuarantees family=A region=ffe1934c280093c181c2b062a05e0908] 2024-11-16T09:59:34,588 DEBUG [RS_FLUSH_OPERATIONS-regionserver/037c58a19ddb:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/ffe1934c280093c181c2b062a05e0908/.tmp/A/fc3e9b5bbfd24a0480896ecf8db38f28 is 175, key is test_row_0/A:col10/1731751172846/Put/seqid=0 2024-11-16T09:59:34,593 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37591 is added to blk_1073742204_1380 (size=31005) 2024-11-16T09:59:34,733 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37877 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=101 2024-11-16T09:59:34,954 DEBUG [RS:0;037c58a19ddb:37267-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/ffe1934c280093c181c2b062a05e0908/.tmp/C/a53a9c0af5e648b2bedbfa0d8cb8a65b as hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/ffe1934c280093c181c2b062a05e0908/C/a53a9c0af5e648b2bedbfa0d8cb8a65b 2024-11-16T09:59:34,959 INFO [RS:0;037c58a19ddb:37267-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in ffe1934c280093c181c2b062a05e0908/C of ffe1934c280093c181c2b062a05e0908 into a53a9c0af5e648b2bedbfa0d8cb8a65b(size=12.0 K), total size for store is 12.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-16T09:59:34,959 DEBUG [RS:0;037c58a19ddb:37267-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for ffe1934c280093c181c2b062a05e0908: 2024-11-16T09:59:34,959 INFO [RS:0;037c58a19ddb:37267-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731751162662.ffe1934c280093c181c2b062a05e0908., storeName=ffe1934c280093c181c2b062a05e0908/C, priority=12, startTime=1731751174496; duration=0sec 2024-11-16T09:59:34,959 DEBUG [RS:0;037c58a19ddb:37267-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-16T09:59:34,959 DEBUG [RS:0;037c58a19ddb:37267-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: ffe1934c280093c181c2b062a05e0908:C 2024-11-16T09:59:34,994 INFO [RS_FLUSH_OPERATIONS-regionserver/037c58a19ddb:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=132, memsize=17.9 K, hasBloomFilter=true, into tmp file hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/ffe1934c280093c181c2b062a05e0908/.tmp/A/fc3e9b5bbfd24a0480896ecf8db38f28 2024-11-16T09:59:35,002 DEBUG [RS_FLUSH_OPERATIONS-regionserver/037c58a19ddb:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/ffe1934c280093c181c2b062a05e0908/.tmp/B/8e4d5c1cfc8548b5b36ca5950cd01fc1 is 50, key is test_row_0/B:col10/1731751172846/Put/seqid=0 2024-11-16T09:59:35,010 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37591 is added to blk_1073742205_1381 (size=12051) 2024-11-16T09:59:35,010 INFO [RS_FLUSH_OPERATIONS-regionserver/037c58a19ddb:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=132 (bloomFilter=true), to=hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/ffe1934c280093c181c2b062a05e0908/.tmp/B/8e4d5c1cfc8548b5b36ca5950cd01fc1 2024-11-16T09:59:35,018 DEBUG [RS_FLUSH_OPERATIONS-regionserver/037c58a19ddb:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/ffe1934c280093c181c2b062a05e0908/.tmp/C/080481f61e514e57b5256885a8981750 is 50, key is test_row_0/C:col10/1731751172846/Put/seqid=0 2024-11-16T09:59:35,025 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37591 is added to blk_1073742206_1382 (size=12051) 2024-11-16T09:59:35,027 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] regionserver.HRegion(8855): Flush requested on ffe1934c280093c181c2b062a05e0908 2024-11-16T09:59:35,027 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731751162662.ffe1934c280093c181c2b062a05e0908. as already flushing 2024-11-16T09:59:35,027 INFO [RS_FLUSH_OPERATIONS-regionserver/037c58a19ddb:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=132 (bloomFilter=true), to=hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/ffe1934c280093c181c2b062a05e0908/.tmp/C/080481f61e514e57b5256885a8981750 2024-11-16T09:59:35,032 DEBUG [RS_FLUSH_OPERATIONS-regionserver/037c58a19ddb:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/ffe1934c280093c181c2b062a05e0908/.tmp/A/fc3e9b5bbfd24a0480896ecf8db38f28 as hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/ffe1934c280093c181c2b062a05e0908/A/fc3e9b5bbfd24a0480896ecf8db38f28 2024-11-16T09:59:35,039 INFO [RS_FLUSH_OPERATIONS-regionserver/037c58a19ddb:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/ffe1934c280093c181c2b062a05e0908/A/fc3e9b5bbfd24a0480896ecf8db38f28, entries=150, sequenceid=132, filesize=30.3 K 2024-11-16T09:59:35,040 DEBUG [RS_FLUSH_OPERATIONS-regionserver/037c58a19ddb:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/ffe1934c280093c181c2b062a05e0908/.tmp/B/8e4d5c1cfc8548b5b36ca5950cd01fc1 as hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/ffe1934c280093c181c2b062a05e0908/B/8e4d5c1cfc8548b5b36ca5950cd01fc1 2024-11-16T09:59:35,044 INFO [RS_FLUSH_OPERATIONS-regionserver/037c58a19ddb:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/ffe1934c280093c181c2b062a05e0908/B/8e4d5c1cfc8548b5b36ca5950cd01fc1, entries=150, sequenceid=132, filesize=11.8 K 2024-11-16T09:59:35,045 DEBUG [RS_FLUSH_OPERATIONS-regionserver/037c58a19ddb:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/ffe1934c280093c181c2b062a05e0908/.tmp/C/080481f61e514e57b5256885a8981750 as hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/ffe1934c280093c181c2b062a05e0908/C/080481f61e514e57b5256885a8981750 2024-11-16T09:59:35,049 INFO [RS_FLUSH_OPERATIONS-regionserver/037c58a19ddb:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/ffe1934c280093c181c2b062a05e0908/C/080481f61e514e57b5256885a8981750, entries=150, sequenceid=132, filesize=11.8 K 2024-11-16T09:59:35,050 INFO [RS_FLUSH_OPERATIONS-regionserver/037c58a19ddb:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~141.33 KB/144720, currentSize=60.38 KB/61830 for ffe1934c280093c181c2b062a05e0908 in 480ms, sequenceid=132, compaction requested=false 2024-11-16T09:59:35,050 DEBUG [RS_FLUSH_OPERATIONS-regionserver/037c58a19ddb:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HRegion(2603): Flush status journal for ffe1934c280093c181c2b062a05e0908: 2024-11-16T09:59:35,050 DEBUG [RS_FLUSH_OPERATIONS-regionserver/037c58a19ddb:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731751162662.ffe1934c280093c181c2b062a05e0908. 2024-11-16T09:59:35,050 DEBUG [RS_FLUSH_OPERATIONS-regionserver/037c58a19ddb:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=102 2024-11-16T09:59:35,050 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37877 {}] master.HMaster(4169): Remote procedure done, pid=102 2024-11-16T09:59:35,050 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] regionserver.HRegion(8855): Flush requested on ffe1934c280093c181c2b062a05e0908 2024-11-16T09:59:35,051 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-16T09:59:35,051 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing ffe1934c280093c181c2b062a05e0908 3/3 column families, dataSize=67.09 KB heapSize=176.53 KB 2024-11-16T09:59:35,055 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=102, resume processing ppid=101 2024-11-16T09:59:35,055 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=102, ppid=101, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 945 msec 2024-11-16T09:59:35,057 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=101, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=101, table=TestAcidGuarantees in 952 msec 2024-11-16T09:59:35,059 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111660cb07e574154cb98add16287a603d40_ffe1934c280093c181c2b062a05e0908 is 50, key is test_row_0/A:col10/1731751175049/Put/seqid=0 2024-11-16T09:59:35,074 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37591 is added to blk_1073742207_1383 (size=12304) 2024-11-16T09:59:35,075 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T09:59:35,079 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111660cb07e574154cb98add16287a603d40_ffe1934c280093c181c2b062a05e0908 to hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111660cb07e574154cb98add16287a603d40_ffe1934c280093c181c2b062a05e0908 2024-11-16T09:59:35,080 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/ffe1934c280093c181c2b062a05e0908/.tmp/A/4de463595afb42a0b38dc1e8ddeca7b7, store: [table=TestAcidGuarantees family=A region=ffe1934c280093c181c2b062a05e0908] 2024-11-16T09:59:35,080 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/ffe1934c280093c181c2b062a05e0908/.tmp/A/4de463595afb42a0b38dc1e8ddeca7b7 is 175, key is test_row_0/A:col10/1731751175049/Put/seqid=0 2024-11-16T09:59:35,087 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37591 is added to blk_1073742208_1384 (size=31105) 2024-11-16T09:59:35,095 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffe1934c280093c181c2b062a05e0908, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T09:59:35,096 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43402 deadline: 1731751235081, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffe1934c280093c181c2b062a05e0908, server=037c58a19ddb,37267,1731751077844 2024-11-16T09:59:35,096 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731751162662.ffe1934c280093c181c2b062a05e0908., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 , the old value is region=TestAcidGuarantees,,1731751162662.ffe1934c280093c181c2b062a05e0908., hostname=037c58a19ddb,37267,1731751077844, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffe1934c280093c181c2b062a05e0908, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:59:35,096 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731751162662.ffe1934c280093c181c2b062a05e0908., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffe1934c280093c181c2b062a05e0908, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:59:35,096 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731751162662.ffe1934c280093c181c2b062a05e0908., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 because the exception is null or not the one we care about 2024-11-16T09:59:35,100 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffe1934c280093c181c2b062a05e0908, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T09:59:35,100 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43332 deadline: 1731751235090, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffe1934c280093c181c2b062a05e0908, server=037c58a19ddb,37267,1731751077844 2024-11-16T09:59:35,101 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffe1934c280093c181c2b062a05e0908, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T09:59:35,101 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731751162662.ffe1934c280093c181c2b062a05e0908., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 , the old value is region=TestAcidGuarantees,,1731751162662.ffe1934c280093c181c2b062a05e0908., hostname=037c58a19ddb,37267,1731751077844, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffe1934c280093c181c2b062a05e0908, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:59:35,101 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731751162662.ffe1934c280093c181c2b062a05e0908., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffe1934c280093c181c2b062a05e0908, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:59:35,101 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731751162662.ffe1934c280093c181c2b062a05e0908., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 because the exception is null or not the one we care about 2024-11-16T09:59:35,101 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43418 deadline: 1731751235091, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffe1934c280093c181c2b062a05e0908, server=037c58a19ddb,37267,1731751077844 2024-11-16T09:59:35,101 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731751162662.ffe1934c280093c181c2b062a05e0908., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 , the old value is region=TestAcidGuarantees,,1731751162662.ffe1934c280093c181c2b062a05e0908., hostname=037c58a19ddb,37267,1731751077844, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffe1934c280093c181c2b062a05e0908, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:59:35,101 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731751162662.ffe1934c280093c181c2b062a05e0908., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffe1934c280093c181c2b062a05e0908, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:59:35,101 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731751162662.ffe1934c280093c181c2b062a05e0908., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 because the exception is null or not the one we care about 2024-11-16T09:59:35,108 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffe1934c280093c181c2b062a05e0908, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T09:59:35,108 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43362 deadline: 1731751235094, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffe1934c280093c181c2b062a05e0908, server=037c58a19ddb,37267,1731751077844 2024-11-16T09:59:35,108 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731751162662.ffe1934c280093c181c2b062a05e0908., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 , the old value is region=TestAcidGuarantees,,1731751162662.ffe1934c280093c181c2b062a05e0908., hostname=037c58a19ddb,37267,1731751077844, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffe1934c280093c181c2b062a05e0908, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:59:35,108 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731751162662.ffe1934c280093c181c2b062a05e0908., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffe1934c280093c181c2b062a05e0908, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:59:35,108 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffe1934c280093c181c2b062a05e0908, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T09:59:35,108 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731751162662.ffe1934c280093c181c2b062a05e0908., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 because the exception is null or not the one we care about 2024-11-16T09:59:35,108 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43416 deadline: 1731751235096, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffe1934c280093c181c2b062a05e0908, server=037c58a19ddb,37267,1731751077844 2024-11-16T09:59:35,108 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731751162662.ffe1934c280093c181c2b062a05e0908., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 , the old value is region=TestAcidGuarantees,,1731751162662.ffe1934c280093c181c2b062a05e0908., hostname=037c58a19ddb,37267,1731751077844, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffe1934c280093c181c2b062a05e0908, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:59:35,108 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731751162662.ffe1934c280093c181c2b062a05e0908., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffe1934c280093c181c2b062a05e0908, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:59:35,108 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731751162662.ffe1934c280093c181c2b062a05e0908., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 because the exception is null or not the one we care about 2024-11-16T09:59:35,207 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffe1934c280093c181c2b062a05e0908, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T09:59:35,207 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43402 deadline: 1731751235203, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffe1934c280093c181c2b062a05e0908, server=037c58a19ddb,37267,1731751077844 2024-11-16T09:59:35,208 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731751162662.ffe1934c280093c181c2b062a05e0908., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 , the old value is region=TestAcidGuarantees,,1731751162662.ffe1934c280093c181c2b062a05e0908., hostname=037c58a19ddb,37267,1731751077844, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffe1934c280093c181c2b062a05e0908, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:59:35,208 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffe1934c280093c181c2b062a05e0908, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T09:59:35,208 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731751162662.ffe1934c280093c181c2b062a05e0908., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffe1934c280093c181c2b062a05e0908, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:59:35,208 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731751162662.ffe1934c280093c181c2b062a05e0908., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 because the exception is null or not the one we care about 2024-11-16T09:59:35,208 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43332 deadline: 1731751235203, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffe1934c280093c181c2b062a05e0908, server=037c58a19ddb,37267,1731751077844 2024-11-16T09:59:35,208 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731751162662.ffe1934c280093c181c2b062a05e0908., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 , the old value is region=TestAcidGuarantees,,1731751162662.ffe1934c280093c181c2b062a05e0908., hostname=037c58a19ddb,37267,1731751077844, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffe1934c280093c181c2b062a05e0908, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:59:35,208 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffe1934c280093c181c2b062a05e0908, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T09:59:35,208 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731751162662.ffe1934c280093c181c2b062a05e0908., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffe1934c280093c181c2b062a05e0908, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:59:35,208 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731751162662.ffe1934c280093c181c2b062a05e0908., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 because the exception is null or not the one we care about 2024-11-16T09:59:35,208 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43418 deadline: 1731751235203, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffe1934c280093c181c2b062a05e0908, server=037c58a19ddb,37267,1731751077844 2024-11-16T09:59:35,208 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731751162662.ffe1934c280093c181c2b062a05e0908., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 , the old value is region=TestAcidGuarantees,,1731751162662.ffe1934c280093c181c2b062a05e0908., hostname=037c58a19ddb,37267,1731751077844, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffe1934c280093c181c2b062a05e0908, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:59:35,208 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731751162662.ffe1934c280093c181c2b062a05e0908., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffe1934c280093c181c2b062a05e0908, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:59:35,208 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731751162662.ffe1934c280093c181c2b062a05e0908., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 because the exception is null or not the one we care about 2024-11-16T09:59:35,218 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffe1934c280093c181c2b062a05e0908, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T09:59:35,218 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43362 deadline: 1731751235212, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffe1934c280093c181c2b062a05e0908, server=037c58a19ddb,37267,1731751077844 2024-11-16T09:59:35,219 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731751162662.ffe1934c280093c181c2b062a05e0908., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 , the old value is region=TestAcidGuarantees,,1731751162662.ffe1934c280093c181c2b062a05e0908., hostname=037c58a19ddb,37267,1731751077844, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffe1934c280093c181c2b062a05e0908, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:59:35,219 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731751162662.ffe1934c280093c181c2b062a05e0908., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffe1934c280093c181c2b062a05e0908, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:59:35,219 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731751162662.ffe1934c280093c181c2b062a05e0908., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 because the exception is null or not the one we care about 2024-11-16T09:59:35,219 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffe1934c280093c181c2b062a05e0908, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T09:59:35,219 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43416 deadline: 1731751235213, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffe1934c280093c181c2b062a05e0908, server=037c58a19ddb,37267,1731751077844 2024-11-16T09:59:35,219 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731751162662.ffe1934c280093c181c2b062a05e0908., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 , the old value is region=TestAcidGuarantees,,1731751162662.ffe1934c280093c181c2b062a05e0908., hostname=037c58a19ddb,37267,1731751077844, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffe1934c280093c181c2b062a05e0908, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:59:35,219 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731751162662.ffe1934c280093c181c2b062a05e0908., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffe1934c280093c181c2b062a05e0908, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:59:35,219 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731751162662.ffe1934c280093c181c2b062a05e0908., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 because the exception is null or not the one we care about 2024-11-16T09:59:35,243 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37877 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=101 2024-11-16T09:59:35,243 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-16T09:59:35,244 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37877 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-16T09:59:35,244 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37877 {}] procedure2.ProcedureExecutor(1139): Stored pid=103, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=103, table=TestAcidGuarantees 2024-11-16T09:59:35,245 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37877 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=103 2024-11-16T09:59:35,245 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=103, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=103, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-16T09:59:35,246 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=103, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=103, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-16T09:59:35,246 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=104, ppid=103, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-16T09:59:35,353 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37877 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=103 2024-11-16T09:59:35,398 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=37267 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=104 2024-11-16T09:59:35,399 DEBUG [RS_FLUSH_OPERATIONS-regionserver/037c58a19ddb:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731751162662.ffe1934c280093c181c2b062a05e0908. 2024-11-16T09:59:35,399 DEBUG [RS_FLUSH_OPERATIONS-regionserver/037c58a19ddb:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731751162662.ffe1934c280093c181c2b062a05e0908. as already flushing 2024-11-16T09:59:35,399 DEBUG [RS_FLUSH_OPERATIONS-regionserver/037c58a19ddb:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731751162662.ffe1934c280093c181c2b062a05e0908. 2024-11-16T09:59:35,399 ERROR [RS_FLUSH_OPERATIONS-regionserver/037c58a19ddb:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] handler.RSProcedureHandler(58): pid=104 java.io.IOException: Unable to complete flush {ENCODED => ffe1934c280093c181c2b062a05e0908, NAME => 'TestAcidGuarantees,,1731751162662.ffe1934c280093c181c2b062a05e0908.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T09:59:35,399 DEBUG [RS_FLUSH_OPERATIONS-regionserver/037c58a19ddb:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=104 java.io.IOException: Unable to complete flush {ENCODED => ffe1934c280093c181c2b062a05e0908, NAME => 'TestAcidGuarantees,,1731751162662.ffe1934c280093c181c2b062a05e0908.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T09:59:35,399 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37877 {}] master.HMaster(4177): Remote procedure failed, pid=104 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ffe1934c280093c181c2b062a05e0908, NAME => 'TestAcidGuarantees,,1731751162662.ffe1934c280093c181c2b062a05e0908.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ffe1934c280093c181c2b062a05e0908, NAME => 'TestAcidGuarantees,,1731751162662.ffe1934c280093c181c2b062a05e0908.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T09:59:35,418 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffe1934c280093c181c2b062a05e0908, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T09:59:35,418 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43402 deadline: 1731751235412, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffe1934c280093c181c2b062a05e0908, server=037c58a19ddb,37267,1731751077844 2024-11-16T09:59:35,418 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffe1934c280093c181c2b062a05e0908, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T09:59:35,418 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43332 deadline: 1731751235413, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffe1934c280093c181c2b062a05e0908, server=037c58a19ddb,37267,1731751077844 2024-11-16T09:59:35,418 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731751162662.ffe1934c280093c181c2b062a05e0908., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 , the old value is region=TestAcidGuarantees,,1731751162662.ffe1934c280093c181c2b062a05e0908., hostname=037c58a19ddb,37267,1731751077844, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffe1934c280093c181c2b062a05e0908, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:59:35,418 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731751162662.ffe1934c280093c181c2b062a05e0908., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffe1934c280093c181c2b062a05e0908, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:59:35,418 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731751162662.ffe1934c280093c181c2b062a05e0908., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 because the exception is null or not the one we care about 2024-11-16T09:59:35,418 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffe1934c280093c181c2b062a05e0908, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T09:59:35,418 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731751162662.ffe1934c280093c181c2b062a05e0908., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 , the old value is region=TestAcidGuarantees,,1731751162662.ffe1934c280093c181c2b062a05e0908., hostname=037c58a19ddb,37267,1731751077844, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffe1934c280093c181c2b062a05e0908, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:59:35,418 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731751162662.ffe1934c280093c181c2b062a05e0908., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffe1934c280093c181c2b062a05e0908, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:59:35,418 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731751162662.ffe1934c280093c181c2b062a05e0908., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 because the exception is null or not the one we care about 2024-11-16T09:59:35,418 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43418 deadline: 1731751235413, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffe1934c280093c181c2b062a05e0908, server=037c58a19ddb,37267,1731751077844 2024-11-16T09:59:35,418 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731751162662.ffe1934c280093c181c2b062a05e0908., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 , the old value is region=TestAcidGuarantees,,1731751162662.ffe1934c280093c181c2b062a05e0908., hostname=037c58a19ddb,37267,1731751077844, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffe1934c280093c181c2b062a05e0908, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:59:35,418 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731751162662.ffe1934c280093c181c2b062a05e0908., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffe1934c280093c181c2b062a05e0908, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:59:35,419 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731751162662.ffe1934c280093c181c2b062a05e0908., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 because the exception is null or not the one we care about 2024-11-16T09:59:35,429 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffe1934c280093c181c2b062a05e0908, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T09:59:35,429 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43362 deadline: 1731751235423, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffe1934c280093c181c2b062a05e0908, server=037c58a19ddb,37267,1731751077844 2024-11-16T09:59:35,429 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731751162662.ffe1934c280093c181c2b062a05e0908., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 , the old value is region=TestAcidGuarantees,,1731751162662.ffe1934c280093c181c2b062a05e0908., hostname=037c58a19ddb,37267,1731751077844, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffe1934c280093c181c2b062a05e0908, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:59:35,429 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffe1934c280093c181c2b062a05e0908, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T09:59:35,429 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731751162662.ffe1934c280093c181c2b062a05e0908., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffe1934c280093c181c2b062a05e0908, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:59:35,429 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731751162662.ffe1934c280093c181c2b062a05e0908., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 because the exception is null or not the one we care about 2024-11-16T09:59:35,429 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43416 deadline: 1731751235423, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffe1934c280093c181c2b062a05e0908, server=037c58a19ddb,37267,1731751077844 2024-11-16T09:59:35,429 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731751162662.ffe1934c280093c181c2b062a05e0908., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 , the old value is region=TestAcidGuarantees,,1731751162662.ffe1934c280093c181c2b062a05e0908., hostname=037c58a19ddb,37267,1731751077844, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffe1934c280093c181c2b062a05e0908, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:59:35,429 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731751162662.ffe1934c280093c181c2b062a05e0908., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffe1934c280093c181c2b062a05e0908, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:59:35,429 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731751162662.ffe1934c280093c181c2b062a05e0908., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 because the exception is null or not the one we care about 2024-11-16T09:59:35,488 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=147, memsize=24.6 K, hasBloomFilter=true, into tmp file hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/ffe1934c280093c181c2b062a05e0908/.tmp/A/4de463595afb42a0b38dc1e8ddeca7b7 2024-11-16T09:59:35,495 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/ffe1934c280093c181c2b062a05e0908/.tmp/B/50a26aec9b7c4e7dbbeffe873e11106f is 50, key is test_row_0/B:col10/1731751175049/Put/seqid=0 2024-11-16T09:59:35,505 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37591 is added to blk_1073742209_1385 (size=12151) 2024-11-16T09:59:35,505 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=147 (bloomFilter=true), to=hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/ffe1934c280093c181c2b062a05e0908/.tmp/B/50a26aec9b7c4e7dbbeffe873e11106f 2024-11-16T09:59:35,513 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/ffe1934c280093c181c2b062a05e0908/.tmp/C/627b560ebad44444b790f9f1742da379 is 50, key is test_row_0/C:col10/1731751175049/Put/seqid=0 2024-11-16T09:59:35,519 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37591 is added to blk_1073742210_1386 (size=12151) 2024-11-16T09:59:35,551 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=37267 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=104 2024-11-16T09:59:35,551 DEBUG [RS_FLUSH_OPERATIONS-regionserver/037c58a19ddb:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731751162662.ffe1934c280093c181c2b062a05e0908. 2024-11-16T09:59:35,552 DEBUG [RS_FLUSH_OPERATIONS-regionserver/037c58a19ddb:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731751162662.ffe1934c280093c181c2b062a05e0908. as already flushing 2024-11-16T09:59:35,552 DEBUG [RS_FLUSH_OPERATIONS-regionserver/037c58a19ddb:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731751162662.ffe1934c280093c181c2b062a05e0908. 2024-11-16T09:59:35,552 ERROR [RS_FLUSH_OPERATIONS-regionserver/037c58a19ddb:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] handler.RSProcedureHandler(58): pid=104 java.io.IOException: Unable to complete flush {ENCODED => ffe1934c280093c181c2b062a05e0908, NAME => 'TestAcidGuarantees,,1731751162662.ffe1934c280093c181c2b062a05e0908.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T09:59:35,552 DEBUG [RS_FLUSH_OPERATIONS-regionserver/037c58a19ddb:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=104 java.io.IOException: Unable to complete flush {ENCODED => ffe1934c280093c181c2b062a05e0908, NAME => 'TestAcidGuarantees,,1731751162662.ffe1934c280093c181c2b062a05e0908.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T09:59:35,552 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37877 {}] master.HMaster(4177): Remote procedure failed, pid=104 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ffe1934c280093c181c2b062a05e0908, NAME => 'TestAcidGuarantees,,1731751162662.ffe1934c280093c181c2b062a05e0908.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ffe1934c280093c181c2b062a05e0908, NAME => 'TestAcidGuarantees,,1731751162662.ffe1934c280093c181c2b062a05e0908.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T09:59:35,562 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37877 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=103 2024-11-16T09:59:35,704 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=37267 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=104 2024-11-16T09:59:35,704 DEBUG [RS_FLUSH_OPERATIONS-regionserver/037c58a19ddb:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731751162662.ffe1934c280093c181c2b062a05e0908. 2024-11-16T09:59:35,704 DEBUG [RS_FLUSH_OPERATIONS-regionserver/037c58a19ddb:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731751162662.ffe1934c280093c181c2b062a05e0908. as already flushing 2024-11-16T09:59:35,705 DEBUG [RS_FLUSH_OPERATIONS-regionserver/037c58a19ddb:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731751162662.ffe1934c280093c181c2b062a05e0908. 2024-11-16T09:59:35,705 ERROR [RS_FLUSH_OPERATIONS-regionserver/037c58a19ddb:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] handler.RSProcedureHandler(58): pid=104 java.io.IOException: Unable to complete flush {ENCODED => ffe1934c280093c181c2b062a05e0908, NAME => 'TestAcidGuarantees,,1731751162662.ffe1934c280093c181c2b062a05e0908.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T09:59:35,705 DEBUG [RS_FLUSH_OPERATIONS-regionserver/037c58a19ddb:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=104 java.io.IOException: Unable to complete flush {ENCODED => ffe1934c280093c181c2b062a05e0908, NAME => 'TestAcidGuarantees,,1731751162662.ffe1934c280093c181c2b062a05e0908.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T09:59:35,705 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37877 {}] master.HMaster(4177): Remote procedure failed, pid=104 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ffe1934c280093c181c2b062a05e0908, NAME => 'TestAcidGuarantees,,1731751162662.ffe1934c280093c181c2b062a05e0908.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ffe1934c280093c181c2b062a05e0908, NAME => 'TestAcidGuarantees,,1731751162662.ffe1934c280093c181c2b062a05e0908.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T09:59:35,730 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffe1934c280093c181c2b062a05e0908, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T09:59:35,731 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43402 deadline: 1731751235723, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffe1934c280093c181c2b062a05e0908, server=037c58a19ddb,37267,1731751077844 2024-11-16T09:59:35,731 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffe1934c280093c181c2b062a05e0908, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T09:59:35,731 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731751162662.ffe1934c280093c181c2b062a05e0908., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 , the old value is region=TestAcidGuarantees,,1731751162662.ffe1934c280093c181c2b062a05e0908., hostname=037c58a19ddb,37267,1731751077844, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffe1934c280093c181c2b062a05e0908, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:59:35,731 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731751162662.ffe1934c280093c181c2b062a05e0908., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffe1934c280093c181c2b062a05e0908, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:59:35,731 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731751162662.ffe1934c280093c181c2b062a05e0908., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 because the exception is null or not the one we care about 2024-11-16T09:59:35,731 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43332 deadline: 1731751235723, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffe1934c280093c181c2b062a05e0908, server=037c58a19ddb,37267,1731751077844 2024-11-16T09:59:35,731 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffe1934c280093c181c2b062a05e0908, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T09:59:35,731 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731751162662.ffe1934c280093c181c2b062a05e0908., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 , the old value is region=TestAcidGuarantees,,1731751162662.ffe1934c280093c181c2b062a05e0908., hostname=037c58a19ddb,37267,1731751077844, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffe1934c280093c181c2b062a05e0908, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:59:35,731 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731751162662.ffe1934c280093c181c2b062a05e0908., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffe1934c280093c181c2b062a05e0908, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:59:35,731 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43418 deadline: 1731751235723, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffe1934c280093c181c2b062a05e0908, server=037c58a19ddb,37267,1731751077844 2024-11-16T09:59:35,731 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731751162662.ffe1934c280093c181c2b062a05e0908., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 because the exception is null or not the one we care about 2024-11-16T09:59:35,731 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731751162662.ffe1934c280093c181c2b062a05e0908., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 , the old value is region=TestAcidGuarantees,,1731751162662.ffe1934c280093c181c2b062a05e0908., hostname=037c58a19ddb,37267,1731751077844, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffe1934c280093c181c2b062a05e0908, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:59:35,731 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731751162662.ffe1934c280093c181c2b062a05e0908., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffe1934c280093c181c2b062a05e0908, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:59:35,731 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731751162662.ffe1934c280093c181c2b062a05e0908., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 because the exception is null or not the one we care about 2024-11-16T09:59:35,742 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffe1934c280093c181c2b062a05e0908, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T09:59:35,742 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43362 deadline: 1731751235733, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffe1934c280093c181c2b062a05e0908, server=037c58a19ddb,37267,1731751077844 2024-11-16T09:59:35,742 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731751162662.ffe1934c280093c181c2b062a05e0908., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 , the old value is region=TestAcidGuarantees,,1731751162662.ffe1934c280093c181c2b062a05e0908., hostname=037c58a19ddb,37267,1731751077844, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffe1934c280093c181c2b062a05e0908, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:59:35,742 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731751162662.ffe1934c280093c181c2b062a05e0908., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffe1934c280093c181c2b062a05e0908, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:59:35,742 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731751162662.ffe1934c280093c181c2b062a05e0908., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 because the exception is null or not the one we care about 2024-11-16T09:59:35,748 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffe1934c280093c181c2b062a05e0908, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T09:59:35,748 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43416 deadline: 1731751235742, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffe1934c280093c181c2b062a05e0908, server=037c58a19ddb,37267,1731751077844 2024-11-16T09:59:35,748 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731751162662.ffe1934c280093c181c2b062a05e0908., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 , the old value is region=TestAcidGuarantees,,1731751162662.ffe1934c280093c181c2b062a05e0908., hostname=037c58a19ddb,37267,1731751077844, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffe1934c280093c181c2b062a05e0908, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:59:35,748 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731751162662.ffe1934c280093c181c2b062a05e0908., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffe1934c280093c181c2b062a05e0908, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:59:35,749 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731751162662.ffe1934c280093c181c2b062a05e0908., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 because the exception is null or not the one we care about 2024-11-16T09:59:35,857 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=37267 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=104 2024-11-16T09:59:35,858 DEBUG [RS_FLUSH_OPERATIONS-regionserver/037c58a19ddb:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731751162662.ffe1934c280093c181c2b062a05e0908. 2024-11-16T09:59:35,858 DEBUG [RS_FLUSH_OPERATIONS-regionserver/037c58a19ddb:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731751162662.ffe1934c280093c181c2b062a05e0908. as already flushing 2024-11-16T09:59:35,858 DEBUG [RS_FLUSH_OPERATIONS-regionserver/037c58a19ddb:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731751162662.ffe1934c280093c181c2b062a05e0908. 2024-11-16T09:59:35,858 ERROR [RS_FLUSH_OPERATIONS-regionserver/037c58a19ddb:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] handler.RSProcedureHandler(58): pid=104 java.io.IOException: Unable to complete flush {ENCODED => ffe1934c280093c181c2b062a05e0908, NAME => 'TestAcidGuarantees,,1731751162662.ffe1934c280093c181c2b062a05e0908.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T09:59:35,858 DEBUG [RS_FLUSH_OPERATIONS-regionserver/037c58a19ddb:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=104 java.io.IOException: Unable to complete flush {ENCODED => ffe1934c280093c181c2b062a05e0908, NAME => 'TestAcidGuarantees,,1731751162662.ffe1934c280093c181c2b062a05e0908.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T09:59:35,859 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37877 {}] master.HMaster(4177): Remote procedure failed, pid=104 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ffe1934c280093c181c2b062a05e0908, NAME => 'TestAcidGuarantees,,1731751162662.ffe1934c280093c181c2b062a05e0908.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ffe1934c280093c181c2b062a05e0908, NAME => 'TestAcidGuarantees,,1731751162662.ffe1934c280093c181c2b062a05e0908.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T09:59:35,872 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37877 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=103 2024-11-16T09:59:35,920 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=147 (bloomFilter=true), to=hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/ffe1934c280093c181c2b062a05e0908/.tmp/C/627b560ebad44444b790f9f1742da379 2024-11-16T09:59:35,924 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/ffe1934c280093c181c2b062a05e0908/.tmp/A/4de463595afb42a0b38dc1e8ddeca7b7 as hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/ffe1934c280093c181c2b062a05e0908/A/4de463595afb42a0b38dc1e8ddeca7b7 2024-11-16T09:59:35,928 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/ffe1934c280093c181c2b062a05e0908/A/4de463595afb42a0b38dc1e8ddeca7b7, entries=150, sequenceid=147, filesize=30.4 K 2024-11-16T09:59:35,928 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/ffe1934c280093c181c2b062a05e0908/.tmp/B/50a26aec9b7c4e7dbbeffe873e11106f as hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/ffe1934c280093c181c2b062a05e0908/B/50a26aec9b7c4e7dbbeffe873e11106f 2024-11-16T09:59:35,932 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/ffe1934c280093c181c2b062a05e0908/B/50a26aec9b7c4e7dbbeffe873e11106f, entries=150, sequenceid=147, filesize=11.9 K 2024-11-16T09:59:35,932 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/ffe1934c280093c181c2b062a05e0908/.tmp/C/627b560ebad44444b790f9f1742da379 as hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/ffe1934c280093c181c2b062a05e0908/C/627b560ebad44444b790f9f1742da379 2024-11-16T09:59:35,936 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/ffe1934c280093c181c2b062a05e0908/C/627b560ebad44444b790f9f1742da379, entries=150, sequenceid=147, filesize=11.9 K 2024-11-16T09:59:35,936 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~73.80 KB/75570, heapSize ~194.06 KB/198720, currentSize=127.47 KB/130530 for ffe1934c280093c181c2b062a05e0908 in 885ms, sequenceid=147, compaction requested=true 2024-11-16T09:59:35,936 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for ffe1934c280093c181c2b062a05e0908: 2024-11-16T09:59:35,937 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store ffe1934c280093c181c2b062a05e0908:A, priority=-2147483648, current under compaction store size is 1 2024-11-16T09:59:35,937 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-16T09:59:35,937 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store ffe1934c280093c181c2b062a05e0908:B, priority=-2147483648, current under compaction store size is 2 2024-11-16T09:59:35,937 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-16T09:59:35,937 DEBUG [RS:0;037c58a19ddb:37267-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-16T09:59:35,937 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store ffe1934c280093c181c2b062a05e0908:C, priority=-2147483648, current under compaction store size is 3 2024-11-16T09:59:35,937 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-16T09:59:35,937 DEBUG [RS:0;037c58a19ddb:37267-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-16T09:59:35,938 DEBUG [RS:0;037c58a19ddb:37267-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36443 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-16T09:59:35,938 DEBUG [RS:0;037c58a19ddb:37267-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 93305 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-16T09:59:35,938 DEBUG [RS:0;037c58a19ddb:37267-longCompactions-0 {}] regionserver.HStore(1541): ffe1934c280093c181c2b062a05e0908/B is initiating minor compaction (all files) 2024-11-16T09:59:35,938 DEBUG [RS:0;037c58a19ddb:37267-shortCompactions-0 {}] regionserver.HStore(1541): ffe1934c280093c181c2b062a05e0908/A is initiating minor compaction (all files) 2024-11-16T09:59:35,938 INFO [RS:0;037c58a19ddb:37267-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of ffe1934c280093c181c2b062a05e0908/B in TestAcidGuarantees,,1731751162662.ffe1934c280093c181c2b062a05e0908. 2024-11-16T09:59:35,938 INFO [RS:0;037c58a19ddb:37267-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of ffe1934c280093c181c2b062a05e0908/A in TestAcidGuarantees,,1731751162662.ffe1934c280093c181c2b062a05e0908. 2024-11-16T09:59:35,938 INFO [RS:0;037c58a19ddb:37267-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/ffe1934c280093c181c2b062a05e0908/B/02659887633940278476ed746fc4b512, hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/ffe1934c280093c181c2b062a05e0908/B/8e4d5c1cfc8548b5b36ca5950cd01fc1, hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/ffe1934c280093c181c2b062a05e0908/B/50a26aec9b7c4e7dbbeffe873e11106f] into tmpdir=hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/ffe1934c280093c181c2b062a05e0908/.tmp, totalSize=35.6 K 2024-11-16T09:59:35,938 INFO [RS:0;037c58a19ddb:37267-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/ffe1934c280093c181c2b062a05e0908/A/b94361fb6b4448359e6a3cb995534553, hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/ffe1934c280093c181c2b062a05e0908/A/fc3e9b5bbfd24a0480896ecf8db38f28, hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/ffe1934c280093c181c2b062a05e0908/A/4de463595afb42a0b38dc1e8ddeca7b7] into tmpdir=hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/ffe1934c280093c181c2b062a05e0908/.tmp, totalSize=91.1 K 2024-11-16T09:59:35,938 INFO [RS:0;037c58a19ddb:37267-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731751162662.ffe1934c280093c181c2b062a05e0908. 2024-11-16T09:59:35,938 DEBUG [RS:0;037c58a19ddb:37267-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731751162662.ffe1934c280093c181c2b062a05e0908. files: [hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/ffe1934c280093c181c2b062a05e0908/A/b94361fb6b4448359e6a3cb995534553, hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/ffe1934c280093c181c2b062a05e0908/A/fc3e9b5bbfd24a0480896ecf8db38f28, hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/ffe1934c280093c181c2b062a05e0908/A/4de463595afb42a0b38dc1e8ddeca7b7] 2024-11-16T09:59:35,938 DEBUG [RS:0;037c58a19ddb:37267-longCompactions-0 {}] compactions.Compactor(225): Compacting 02659887633940278476ed746fc4b512, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=119, earliestPutTs=1731751172157 2024-11-16T09:59:35,938 DEBUG [RS:0;037c58a19ddb:37267-shortCompactions-0 {}] compactions.Compactor(225): Compacting b94361fb6b4448359e6a3cb995534553, keycount=150, bloomtype=ROW, size=30.5 K, encoding=NONE, compression=NONE, seqNum=119, earliestPutTs=1731751172157 2024-11-16T09:59:35,938 DEBUG [RS:0;037c58a19ddb:37267-longCompactions-0 {}] compactions.Compactor(225): Compacting 8e4d5c1cfc8548b5b36ca5950cd01fc1, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=132, earliestPutTs=1731751172844 2024-11-16T09:59:35,938 DEBUG [RS:0;037c58a19ddb:37267-shortCompactions-0 {}] compactions.Compactor(225): Compacting fc3e9b5bbfd24a0480896ecf8db38f28, keycount=150, bloomtype=ROW, size=30.3 K, encoding=NONE, compression=NONE, seqNum=132, earliestPutTs=1731751172844 2024-11-16T09:59:35,939 DEBUG [RS:0;037c58a19ddb:37267-shortCompactions-0 {}] compactions.Compactor(225): Compacting 4de463595afb42a0b38dc1e8ddeca7b7, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=147, earliestPutTs=1731751175027 2024-11-16T09:59:35,939 DEBUG [RS:0;037c58a19ddb:37267-longCompactions-0 {}] compactions.Compactor(225): Compacting 50a26aec9b7c4e7dbbeffe873e11106f, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=147, earliestPutTs=1731751175027 2024-11-16T09:59:35,944 INFO [RS:0;037c58a19ddb:37267-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=ffe1934c280093c181c2b062a05e0908] 2024-11-16T09:59:35,945 INFO [RS:0;037c58a19ddb:37267-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): ffe1934c280093c181c2b062a05e0908#B#compaction#329 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-16T09:59:35,946 DEBUG [RS:0;037c58a19ddb:37267-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/ffe1934c280093c181c2b062a05e0908/.tmp/B/3b3507fbba454d93a149ba91b104a66e is 50, key is test_row_0/B:col10/1731751175049/Put/seqid=0 2024-11-16T09:59:35,946 DEBUG [RS:0;037c58a19ddb:37267-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e2024111655814f70355f413cac62199599aabe6f_ffe1934c280093c181c2b062a05e0908 store=[table=TestAcidGuarantees family=A region=ffe1934c280093c181c2b062a05e0908] 2024-11-16T09:59:35,948 DEBUG [RS:0;037c58a19ddb:37267-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e2024111655814f70355f413cac62199599aabe6f_ffe1934c280093c181c2b062a05e0908, store=[table=TestAcidGuarantees family=A region=ffe1934c280093c181c2b062a05e0908] 2024-11-16T09:59:35,948 DEBUG [RS:0;037c58a19ddb:37267-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111655814f70355f413cac62199599aabe6f_ffe1934c280093c181c2b062a05e0908 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=ffe1934c280093c181c2b062a05e0908] 2024-11-16T09:59:35,951 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37591 is added to blk_1073742211_1387 (size=12493) 2024-11-16T09:59:35,957 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37591 is added to blk_1073742212_1388 (size=4469) 2024-11-16T09:59:36,010 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=37267 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=104 2024-11-16T09:59:36,011 DEBUG [RS_FLUSH_OPERATIONS-regionserver/037c58a19ddb:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731751162662.ffe1934c280093c181c2b062a05e0908. 2024-11-16T09:59:36,011 INFO [RS_FLUSH_OPERATIONS-regionserver/037c58a19ddb:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HRegion(2902): Flushing ffe1934c280093c181c2b062a05e0908 3/3 column families, dataSize=127.47 KB heapSize=334.73 KB 2024-11-16T09:59:36,018 DEBUG [RS_FLUSH_OPERATIONS-regionserver/037c58a19ddb:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241116d8158e3a222d4016baa85be596ba0e39_ffe1934c280093c181c2b062a05e0908 is 50, key is test_row_0/A:col10/1731751175089/Put/seqid=0 2024-11-16T09:59:36,024 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37591 is added to blk_1073742213_1389 (size=12304) 2024-11-16T09:59:36,245 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] regionserver.HRegion(8855): Flush requested on ffe1934c280093c181c2b062a05e0908 2024-11-16T09:59:36,245 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731751162662.ffe1934c280093c181c2b062a05e0908. as already flushing 2024-11-16T09:59:36,270 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffe1934c280093c181c2b062a05e0908, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T09:59:36,270 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43402 deadline: 1731751236259, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffe1934c280093c181c2b062a05e0908, server=037c58a19ddb,37267,1731751077844 2024-11-16T09:59:36,270 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731751162662.ffe1934c280093c181c2b062a05e0908., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 , the old value is region=TestAcidGuarantees,,1731751162662.ffe1934c280093c181c2b062a05e0908., hostname=037c58a19ddb,37267,1731751077844, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffe1934c280093c181c2b062a05e0908, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:59:36,270 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731751162662.ffe1934c280093c181c2b062a05e0908., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffe1934c280093c181c2b062a05e0908, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:59:36,270 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731751162662.ffe1934c280093c181c2b062a05e0908., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 because the exception is null or not the one we care about 2024-11-16T09:59:36,271 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffe1934c280093c181c2b062a05e0908, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T09:59:36,272 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43362 deadline: 1731751236260, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffe1934c280093c181c2b062a05e0908, server=037c58a19ddb,37267,1731751077844 2024-11-16T09:59:36,272 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffe1934c280093c181c2b062a05e0908, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T09:59:36,272 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731751162662.ffe1934c280093c181c2b062a05e0908., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 , the old value is region=TestAcidGuarantees,,1731751162662.ffe1934c280093c181c2b062a05e0908., hostname=037c58a19ddb,37267,1731751077844, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffe1934c280093c181c2b062a05e0908, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:59:36,272 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731751162662.ffe1934c280093c181c2b062a05e0908., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffe1934c280093c181c2b062a05e0908, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:59:36,272 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731751162662.ffe1934c280093c181c2b062a05e0908., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 because the exception is null or not the one we care about 2024-11-16T09:59:36,272 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43416 deadline: 1731751236261, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffe1934c280093c181c2b062a05e0908, server=037c58a19ddb,37267,1731751077844 2024-11-16T09:59:36,272 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731751162662.ffe1934c280093c181c2b062a05e0908., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 , the old value is region=TestAcidGuarantees,,1731751162662.ffe1934c280093c181c2b062a05e0908., hostname=037c58a19ddb,37267,1731751077844, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffe1934c280093c181c2b062a05e0908, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:59:36,272 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffe1934c280093c181c2b062a05e0908, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T09:59:36,272 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731751162662.ffe1934c280093c181c2b062a05e0908., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffe1934c280093c181c2b062a05e0908, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:59:36,272 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731751162662.ffe1934c280093c181c2b062a05e0908., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 because the exception is null or not the one we care about 2024-11-16T09:59:36,272 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43418 deadline: 1731751236270, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffe1934c280093c181c2b062a05e0908, server=037c58a19ddb,37267,1731751077844 2024-11-16T09:59:36,273 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731751162662.ffe1934c280093c181c2b062a05e0908., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 , the old value is region=TestAcidGuarantees,,1731751162662.ffe1934c280093c181c2b062a05e0908., hostname=037c58a19ddb,37267,1731751077844, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffe1934c280093c181c2b062a05e0908, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:59:36,273 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731751162662.ffe1934c280093c181c2b062a05e0908., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffe1934c280093c181c2b062a05e0908, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:59:36,273 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffe1934c280093c181c2b062a05e0908, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T09:59:36,273 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731751162662.ffe1934c280093c181c2b062a05e0908., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 because the exception is null or not the one we care about 2024-11-16T09:59:36,273 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43332 deadline: 1731751236270, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffe1934c280093c181c2b062a05e0908, server=037c58a19ddb,37267,1731751077844 2024-11-16T09:59:36,273 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731751162662.ffe1934c280093c181c2b062a05e0908., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 , the old value is region=TestAcidGuarantees,,1731751162662.ffe1934c280093c181c2b062a05e0908., hostname=037c58a19ddb,37267,1731751077844, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffe1934c280093c181c2b062a05e0908, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:59:36,273 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731751162662.ffe1934c280093c181c2b062a05e0908., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffe1934c280093c181c2b062a05e0908, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:59:36,273 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731751162662.ffe1934c280093c181c2b062a05e0908., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 because the exception is null or not the one we care about 2024-11-16T09:59:36,356 DEBUG [RS:0;037c58a19ddb:37267-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/ffe1934c280093c181c2b062a05e0908/.tmp/B/3b3507fbba454d93a149ba91b104a66e as hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/ffe1934c280093c181c2b062a05e0908/B/3b3507fbba454d93a149ba91b104a66e 2024-11-16T09:59:36,359 INFO [RS:0;037c58a19ddb:37267-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): ffe1934c280093c181c2b062a05e0908#A#compaction#330 average throughput is 0.06 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-16T09:59:36,360 DEBUG [RS:0;037c58a19ddb:37267-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/ffe1934c280093c181c2b062a05e0908/.tmp/A/4f3739c53ae949579bcbf3fa1d7bcc34 is 175, key is test_row_0/A:col10/1731751175049/Put/seqid=0 2024-11-16T09:59:36,362 INFO [RS:0;037c58a19ddb:37267-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in ffe1934c280093c181c2b062a05e0908/B of ffe1934c280093c181c2b062a05e0908 into 3b3507fbba454d93a149ba91b104a66e(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-16T09:59:36,362 DEBUG [RS:0;037c58a19ddb:37267-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for ffe1934c280093c181c2b062a05e0908: 2024-11-16T09:59:36,362 INFO [RS:0;037c58a19ddb:37267-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731751162662.ffe1934c280093c181c2b062a05e0908., storeName=ffe1934c280093c181c2b062a05e0908/B, priority=13, startTime=1731751175937; duration=0sec 2024-11-16T09:59:36,362 DEBUG [RS:0;037c58a19ddb:37267-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-16T09:59:36,362 DEBUG [RS:0;037c58a19ddb:37267-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: ffe1934c280093c181c2b062a05e0908:B 2024-11-16T09:59:36,362 DEBUG [RS:0;037c58a19ddb:37267-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-16T09:59:36,363 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37591 is added to blk_1073742214_1390 (size=31447) 2024-11-16T09:59:36,363 DEBUG [RS:0;037c58a19ddb:37267-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36443 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-16T09:59:36,363 DEBUG [RS:0;037c58a19ddb:37267-longCompactions-0 {}] regionserver.HStore(1541): ffe1934c280093c181c2b062a05e0908/C is initiating minor compaction (all files) 2024-11-16T09:59:36,364 INFO [RS:0;037c58a19ddb:37267-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of ffe1934c280093c181c2b062a05e0908/C in TestAcidGuarantees,,1731751162662.ffe1934c280093c181c2b062a05e0908. 2024-11-16T09:59:36,364 INFO [RS:0;037c58a19ddb:37267-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/ffe1934c280093c181c2b062a05e0908/C/a53a9c0af5e648b2bedbfa0d8cb8a65b, hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/ffe1934c280093c181c2b062a05e0908/C/080481f61e514e57b5256885a8981750, hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/ffe1934c280093c181c2b062a05e0908/C/627b560ebad44444b790f9f1742da379] into tmpdir=hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/ffe1934c280093c181c2b062a05e0908/.tmp, totalSize=35.6 K 2024-11-16T09:59:36,364 DEBUG [RS:0;037c58a19ddb:37267-longCompactions-0 {}] compactions.Compactor(225): Compacting a53a9c0af5e648b2bedbfa0d8cb8a65b, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=119, earliestPutTs=1731751172157 2024-11-16T09:59:36,364 DEBUG [RS:0;037c58a19ddb:37267-longCompactions-0 {}] compactions.Compactor(225): Compacting 080481f61e514e57b5256885a8981750, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=132, earliestPutTs=1731751172844 2024-11-16T09:59:36,365 DEBUG [RS:0;037c58a19ddb:37267-longCompactions-0 {}] compactions.Compactor(225): Compacting 627b560ebad44444b790f9f1742da379, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=147, earliestPutTs=1731751175027 2024-11-16T09:59:36,370 INFO [RS:0;037c58a19ddb:37267-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): ffe1934c280093c181c2b062a05e0908#C#compaction#332 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-16T09:59:36,370 DEBUG [RS:0;037c58a19ddb:37267-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/ffe1934c280093c181c2b062a05e0908/.tmp/C/ed38a46f975e446c9c74ecd764de8df0 is 50, key is test_row_0/C:col10/1731751175049/Put/seqid=0 2024-11-16T09:59:36,374 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37591 is added to blk_1073742215_1391 (size=12493) 2024-11-16T09:59:36,381 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffe1934c280093c181c2b062a05e0908, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T09:59:36,381 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43402 deadline: 1731751236373, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffe1934c280093c181c2b062a05e0908, server=037c58a19ddb,37267,1731751077844 2024-11-16T09:59:36,381 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731751162662.ffe1934c280093c181c2b062a05e0908., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 , the old value is region=TestAcidGuarantees,,1731751162662.ffe1934c280093c181c2b062a05e0908., hostname=037c58a19ddb,37267,1731751077844, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffe1934c280093c181c2b062a05e0908, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:59:36,382 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731751162662.ffe1934c280093c181c2b062a05e0908., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffe1934c280093c181c2b062a05e0908, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:59:36,382 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731751162662.ffe1934c280093c181c2b062a05e0908., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 because the exception is null or not the one we care about 2024-11-16T09:59:36,383 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37877 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=103 2024-11-16T09:59:36,392 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffe1934c280093c181c2b062a05e0908, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T09:59:36,392 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43362 deadline: 1731751236383, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffe1934c280093c181c2b062a05e0908, server=037c58a19ddb,37267,1731751077844 2024-11-16T09:59:36,392 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffe1934c280093c181c2b062a05e0908, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T09:59:36,392 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731751162662.ffe1934c280093c181c2b062a05e0908., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 , the old value is region=TestAcidGuarantees,,1731751162662.ffe1934c280093c181c2b062a05e0908., hostname=037c58a19ddb,37267,1731751077844, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffe1934c280093c181c2b062a05e0908, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:59:36,393 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43416 deadline: 1731751236383, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffe1934c280093c181c2b062a05e0908, server=037c58a19ddb,37267,1731751077844 2024-11-16T09:59:36,393 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731751162662.ffe1934c280093c181c2b062a05e0908., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffe1934c280093c181c2b062a05e0908, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:59:36,393 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731751162662.ffe1934c280093c181c2b062a05e0908., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 because the exception is null or not the one we care about 2024-11-16T09:59:36,393 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffe1934c280093c181c2b062a05e0908, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T09:59:36,393 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731751162662.ffe1934c280093c181c2b062a05e0908., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 , the old value is region=TestAcidGuarantees,,1731751162662.ffe1934c280093c181c2b062a05e0908., hostname=037c58a19ddb,37267,1731751077844, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffe1934c280093c181c2b062a05e0908, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:59:36,393 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731751162662.ffe1934c280093c181c2b062a05e0908., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffe1934c280093c181c2b062a05e0908, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:59:36,393 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731751162662.ffe1934c280093c181c2b062a05e0908., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 because the exception is null or not the one we care about 2024-11-16T09:59:36,393 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43418 deadline: 1731751236383, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffe1934c280093c181c2b062a05e0908, server=037c58a19ddb,37267,1731751077844 2024-11-16T09:59:36,393 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffe1934c280093c181c2b062a05e0908, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T09:59:36,393 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731751162662.ffe1934c280093c181c2b062a05e0908., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 , the old value is region=TestAcidGuarantees,,1731751162662.ffe1934c280093c181c2b062a05e0908., hostname=037c58a19ddb,37267,1731751077844, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffe1934c280093c181c2b062a05e0908, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:59:36,393 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731751162662.ffe1934c280093c181c2b062a05e0908., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffe1934c280093c181c2b062a05e0908, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:59:36,393 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43332 deadline: 1731751236383, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffe1934c280093c181c2b062a05e0908, server=037c58a19ddb,37267,1731751077844 2024-11-16T09:59:36,393 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731751162662.ffe1934c280093c181c2b062a05e0908., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 because the exception is null or not the one we care about 2024-11-16T09:59:36,393 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731751162662.ffe1934c280093c181c2b062a05e0908., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 , the old value is region=TestAcidGuarantees,,1731751162662.ffe1934c280093c181c2b062a05e0908., hostname=037c58a19ddb,37267,1731751077844, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffe1934c280093c181c2b062a05e0908, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:59:36,393 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731751162662.ffe1934c280093c181c2b062a05e0908., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffe1934c280093c181c2b062a05e0908, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:59:36,393 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731751162662.ffe1934c280093c181c2b062a05e0908., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 because the exception is null or not the one we care about 2024-11-16T09:59:36,425 DEBUG [RS_FLUSH_OPERATIONS-regionserver/037c58a19ddb:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T09:59:36,428 INFO [RS_FLUSH_OPERATIONS-regionserver/037c58a19ddb:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241116d8158e3a222d4016baa85be596ba0e39_ffe1934c280093c181c2b062a05e0908 to hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241116d8158e3a222d4016baa85be596ba0e39_ffe1934c280093c181c2b062a05e0908 2024-11-16T09:59:36,429 DEBUG [RS_FLUSH_OPERATIONS-regionserver/037c58a19ddb:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/ffe1934c280093c181c2b062a05e0908/.tmp/A/ee2723f2d8864f99892f888682a906e1, store: [table=TestAcidGuarantees family=A region=ffe1934c280093c181c2b062a05e0908] 2024-11-16T09:59:36,429 DEBUG [RS_FLUSH_OPERATIONS-regionserver/037c58a19ddb:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/ffe1934c280093c181c2b062a05e0908/.tmp/A/ee2723f2d8864f99892f888682a906e1 is 175, key is test_row_0/A:col10/1731751175089/Put/seqid=0 2024-11-16T09:59:36,433 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37591 is added to blk_1073742216_1392 (size=31105) 2024-11-16T09:59:36,599 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffe1934c280093c181c2b062a05e0908, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T09:59:36,599 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43402 deadline: 1731751236593, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffe1934c280093c181c2b062a05e0908, server=037c58a19ddb,37267,1731751077844 2024-11-16T09:59:36,599 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731751162662.ffe1934c280093c181c2b062a05e0908., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 , the old value is region=TestAcidGuarantees,,1731751162662.ffe1934c280093c181c2b062a05e0908., hostname=037c58a19ddb,37267,1731751077844, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffe1934c280093c181c2b062a05e0908, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:59:36,599 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731751162662.ffe1934c280093c181c2b062a05e0908., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffe1934c280093c181c2b062a05e0908, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:59:36,599 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731751162662.ffe1934c280093c181c2b062a05e0908., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 because the exception is null or not the one we care about 2024-11-16T09:59:36,610 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffe1934c280093c181c2b062a05e0908, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T09:59:36,610 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43362 deadline: 1731751236603, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffe1934c280093c181c2b062a05e0908, server=037c58a19ddb,37267,1731751077844 2024-11-16T09:59:36,610 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffe1934c280093c181c2b062a05e0908, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T09:59:36,610 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43416 deadline: 1731751236603, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffe1934c280093c181c2b062a05e0908, server=037c58a19ddb,37267,1731751077844 2024-11-16T09:59:36,610 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731751162662.ffe1934c280093c181c2b062a05e0908., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 , the old value is region=TestAcidGuarantees,,1731751162662.ffe1934c280093c181c2b062a05e0908., hostname=037c58a19ddb,37267,1731751077844, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffe1934c280093c181c2b062a05e0908, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:59:36,610 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731751162662.ffe1934c280093c181c2b062a05e0908., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffe1934c280093c181c2b062a05e0908, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:59:36,610 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731751162662.ffe1934c280093c181c2b062a05e0908., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 because the exception is null or not the one we care about 2024-11-16T09:59:36,610 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731751162662.ffe1934c280093c181c2b062a05e0908., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 , the old value is region=TestAcidGuarantees,,1731751162662.ffe1934c280093c181c2b062a05e0908., hostname=037c58a19ddb,37267,1731751077844, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffe1934c280093c181c2b062a05e0908, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:59:36,610 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffe1934c280093c181c2b062a05e0908, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T09:59:36,611 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731751162662.ffe1934c280093c181c2b062a05e0908., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffe1934c280093c181c2b062a05e0908, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:59:36,611 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731751162662.ffe1934c280093c181c2b062a05e0908., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 because the exception is null or not the one we care about 2024-11-16T09:59:36,611 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43418 deadline: 1731751236603, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffe1934c280093c181c2b062a05e0908, server=037c58a19ddb,37267,1731751077844 2024-11-16T09:59:36,611 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731751162662.ffe1934c280093c181c2b062a05e0908., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 , the old value is region=TestAcidGuarantees,,1731751162662.ffe1934c280093c181c2b062a05e0908., hostname=037c58a19ddb,37267,1731751077844, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffe1934c280093c181c2b062a05e0908, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:59:36,611 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731751162662.ffe1934c280093c181c2b062a05e0908., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffe1934c280093c181c2b062a05e0908, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:59:36,611 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffe1934c280093c181c2b062a05e0908, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T09:59:36,611 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731751162662.ffe1934c280093c181c2b062a05e0908., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 because the exception is null or not the one we care about 2024-11-16T09:59:36,611 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43332 deadline: 1731751236603, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffe1934c280093c181c2b062a05e0908, server=037c58a19ddb,37267,1731751077844 2024-11-16T09:59:36,611 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731751162662.ffe1934c280093c181c2b062a05e0908., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 , the old value is region=TestAcidGuarantees,,1731751162662.ffe1934c280093c181c2b062a05e0908., hostname=037c58a19ddb,37267,1731751077844, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffe1934c280093c181c2b062a05e0908, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:59:36,611 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731751162662.ffe1934c280093c181c2b062a05e0908., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffe1934c280093c181c2b062a05e0908, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:59:36,611 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731751162662.ffe1934c280093c181c2b062a05e0908., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 because the exception is null or not the one we care about 2024-11-16T09:59:36,768 DEBUG [RS:0;037c58a19ddb:37267-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/ffe1934c280093c181c2b062a05e0908/.tmp/A/4f3739c53ae949579bcbf3fa1d7bcc34 as hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/ffe1934c280093c181c2b062a05e0908/A/4f3739c53ae949579bcbf3fa1d7bcc34 2024-11-16T09:59:36,772 INFO [RS:0;037c58a19ddb:37267-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in ffe1934c280093c181c2b062a05e0908/A of ffe1934c280093c181c2b062a05e0908 into 4f3739c53ae949579bcbf3fa1d7bcc34(size=30.7 K), total size for store is 30.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-16T09:59:36,772 DEBUG [RS:0;037c58a19ddb:37267-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for ffe1934c280093c181c2b062a05e0908: 2024-11-16T09:59:36,772 INFO [RS:0;037c58a19ddb:37267-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731751162662.ffe1934c280093c181c2b062a05e0908., storeName=ffe1934c280093c181c2b062a05e0908/A, priority=13, startTime=1731751175937; duration=0sec 2024-11-16T09:59:36,772 DEBUG [RS:0;037c58a19ddb:37267-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-16T09:59:36,772 DEBUG [RS:0;037c58a19ddb:37267-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: ffe1934c280093c181c2b062a05e0908:A 2024-11-16T09:59:36,778 DEBUG [RS:0;037c58a19ddb:37267-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/ffe1934c280093c181c2b062a05e0908/.tmp/C/ed38a46f975e446c9c74ecd764de8df0 as hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/ffe1934c280093c181c2b062a05e0908/C/ed38a46f975e446c9c74ecd764de8df0 2024-11-16T09:59:36,782 INFO [RS:0;037c58a19ddb:37267-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in ffe1934c280093c181c2b062a05e0908/C of ffe1934c280093c181c2b062a05e0908 into ed38a46f975e446c9c74ecd764de8df0(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-16T09:59:36,782 DEBUG [RS:0;037c58a19ddb:37267-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for ffe1934c280093c181c2b062a05e0908: 2024-11-16T09:59:36,782 INFO [RS:0;037c58a19ddb:37267-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731751162662.ffe1934c280093c181c2b062a05e0908., storeName=ffe1934c280093c181c2b062a05e0908/C, priority=13, startTime=1731751175937; duration=0sec 2024-11-16T09:59:36,782 DEBUG [RS:0;037c58a19ddb:37267-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-16T09:59:36,782 DEBUG [RS:0;037c58a19ddb:37267-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: ffe1934c280093c181c2b062a05e0908:C 2024-11-16T09:59:36,833 INFO [RS_FLUSH_OPERATIONS-regionserver/037c58a19ddb:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=169, memsize=42.5 K, hasBloomFilter=true, into tmp file hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/ffe1934c280093c181c2b062a05e0908/.tmp/A/ee2723f2d8864f99892f888682a906e1 2024-11-16T09:59:36,841 DEBUG [RS_FLUSH_OPERATIONS-regionserver/037c58a19ddb:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/ffe1934c280093c181c2b062a05e0908/.tmp/B/ea96bd3b9778485a8591866303bed84e is 50, key is test_row_0/B:col10/1731751175089/Put/seqid=0 2024-11-16T09:59:36,844 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37591 is added to blk_1073742217_1393 (size=12151) 2024-11-16T09:59:36,845 INFO [RS_FLUSH_OPERATIONS-regionserver/037c58a19ddb:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=169 (bloomFilter=true), to=hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/ffe1934c280093c181c2b062a05e0908/.tmp/B/ea96bd3b9778485a8591866303bed84e 2024-11-16T09:59:36,852 DEBUG [RS_FLUSH_OPERATIONS-regionserver/037c58a19ddb:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/ffe1934c280093c181c2b062a05e0908/.tmp/C/6d07f7417b00489c94662f29698743f1 is 50, key is test_row_0/C:col10/1731751175089/Put/seqid=0 2024-11-16T09:59:36,860 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37591 is added to blk_1073742218_1394 (size=12151) 2024-11-16T09:59:36,909 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffe1934c280093c181c2b062a05e0908, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T09:59:36,910 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43402 deadline: 1731751236902, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffe1934c280093c181c2b062a05e0908, server=037c58a19ddb,37267,1731751077844 2024-11-16T09:59:36,910 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731751162662.ffe1934c280093c181c2b062a05e0908., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 , the old value is region=TestAcidGuarantees,,1731751162662.ffe1934c280093c181c2b062a05e0908., hostname=037c58a19ddb,37267,1731751077844, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffe1934c280093c181c2b062a05e0908, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:59:36,910 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731751162662.ffe1934c280093c181c2b062a05e0908., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffe1934c280093c181c2b062a05e0908, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:59:36,910 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731751162662.ffe1934c280093c181c2b062a05e0908., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 because the exception is null or not the one we care about 2024-11-16T09:59:36,920 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffe1934c280093c181c2b062a05e0908, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T09:59:36,920 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43362 deadline: 1731751236913, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffe1934c280093c181c2b062a05e0908, server=037c58a19ddb,37267,1731751077844 2024-11-16T09:59:36,920 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731751162662.ffe1934c280093c181c2b062a05e0908., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 , the old value is region=TestAcidGuarantees,,1731751162662.ffe1934c280093c181c2b062a05e0908., hostname=037c58a19ddb,37267,1731751077844, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffe1934c280093c181c2b062a05e0908, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:59:36,920 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731751162662.ffe1934c280093c181c2b062a05e0908., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffe1934c280093c181c2b062a05e0908, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:59:36,920 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffe1934c280093c181c2b062a05e0908, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T09:59:36,920 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731751162662.ffe1934c280093c181c2b062a05e0908., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 because the exception is null or not the one we care about 2024-11-16T09:59:36,920 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43416 deadline: 1731751236913, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffe1934c280093c181c2b062a05e0908, server=037c58a19ddb,37267,1731751077844 2024-11-16T09:59:36,921 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731751162662.ffe1934c280093c181c2b062a05e0908., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 , the old value is region=TestAcidGuarantees,,1731751162662.ffe1934c280093c181c2b062a05e0908., hostname=037c58a19ddb,37267,1731751077844, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffe1934c280093c181c2b062a05e0908, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:59:36,921 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731751162662.ffe1934c280093c181c2b062a05e0908., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffe1934c280093c181c2b062a05e0908, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:59:36,921 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731751162662.ffe1934c280093c181c2b062a05e0908., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 because the exception is null or not the one we care about 2024-11-16T09:59:36,930 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffe1934c280093c181c2b062a05e0908, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T09:59:36,931 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43418 deadline: 1731751236923, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffe1934c280093c181c2b062a05e0908, server=037c58a19ddb,37267,1731751077844 2024-11-16T09:59:36,931 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffe1934c280093c181c2b062a05e0908, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T09:59:36,931 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731751162662.ffe1934c280093c181c2b062a05e0908., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 , the old value is region=TestAcidGuarantees,,1731751162662.ffe1934c280093c181c2b062a05e0908., hostname=037c58a19ddb,37267,1731751077844, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffe1934c280093c181c2b062a05e0908, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:59:36,931 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731751162662.ffe1934c280093c181c2b062a05e0908., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffe1934c280093c181c2b062a05e0908, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:59:36,931 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43332 deadline: 1731751236923, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffe1934c280093c181c2b062a05e0908, server=037c58a19ddb,37267,1731751077844 2024-11-16T09:59:36,931 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731751162662.ffe1934c280093c181c2b062a05e0908., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 because the exception is null or not the one we care about 2024-11-16T09:59:36,931 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731751162662.ffe1934c280093c181c2b062a05e0908., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 , the old value is region=TestAcidGuarantees,,1731751162662.ffe1934c280093c181c2b062a05e0908., hostname=037c58a19ddb,37267,1731751077844, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffe1934c280093c181c2b062a05e0908, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:59:36,931 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731751162662.ffe1934c280093c181c2b062a05e0908., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffe1934c280093c181c2b062a05e0908, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:59:36,931 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731751162662.ffe1934c280093c181c2b062a05e0908., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 because the exception is null or not the one we care about 2024-11-16T09:59:37,261 INFO [RS_FLUSH_OPERATIONS-regionserver/037c58a19ddb:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=169 (bloomFilter=true), to=hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/ffe1934c280093c181c2b062a05e0908/.tmp/C/6d07f7417b00489c94662f29698743f1 2024-11-16T09:59:37,265 DEBUG [RS_FLUSH_OPERATIONS-regionserver/037c58a19ddb:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/ffe1934c280093c181c2b062a05e0908/.tmp/A/ee2723f2d8864f99892f888682a906e1 as hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/ffe1934c280093c181c2b062a05e0908/A/ee2723f2d8864f99892f888682a906e1 2024-11-16T09:59:37,268 INFO [RS_FLUSH_OPERATIONS-regionserver/037c58a19ddb:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/ffe1934c280093c181c2b062a05e0908/A/ee2723f2d8864f99892f888682a906e1, entries=150, sequenceid=169, filesize=30.4 K 2024-11-16T09:59:37,269 DEBUG [RS_FLUSH_OPERATIONS-regionserver/037c58a19ddb:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/ffe1934c280093c181c2b062a05e0908/.tmp/B/ea96bd3b9778485a8591866303bed84e as hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/ffe1934c280093c181c2b062a05e0908/B/ea96bd3b9778485a8591866303bed84e 2024-11-16T09:59:37,272 INFO [RS_FLUSH_OPERATIONS-regionserver/037c58a19ddb:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/ffe1934c280093c181c2b062a05e0908/B/ea96bd3b9778485a8591866303bed84e, entries=150, sequenceid=169, filesize=11.9 K 2024-11-16T09:59:37,273 DEBUG [RS_FLUSH_OPERATIONS-regionserver/037c58a19ddb:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/ffe1934c280093c181c2b062a05e0908/.tmp/C/6d07f7417b00489c94662f29698743f1 as hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/ffe1934c280093c181c2b062a05e0908/C/6d07f7417b00489c94662f29698743f1 2024-11-16T09:59:37,276 INFO [RS_FLUSH_OPERATIONS-regionserver/037c58a19ddb:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/ffe1934c280093c181c2b062a05e0908/C/6d07f7417b00489c94662f29698743f1, entries=150, sequenceid=169, filesize=11.9 K 2024-11-16T09:59:37,277 INFO [RS_FLUSH_OPERATIONS-regionserver/037c58a19ddb:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HRegion(3140): Finished flush of dataSize ~127.47 KB/130530, heapSize ~334.69 KB/342720, currentSize=73.80 KB/75570 for ffe1934c280093c181c2b062a05e0908 in 1266ms, sequenceid=169, compaction requested=false 2024-11-16T09:59:37,277 DEBUG [RS_FLUSH_OPERATIONS-regionserver/037c58a19ddb:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HRegion(2603): Flush status journal for ffe1934c280093c181c2b062a05e0908: 2024-11-16T09:59:37,277 DEBUG [RS_FLUSH_OPERATIONS-regionserver/037c58a19ddb:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731751162662.ffe1934c280093c181c2b062a05e0908. 2024-11-16T09:59:37,277 DEBUG [RS_FLUSH_OPERATIONS-regionserver/037c58a19ddb:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=104 2024-11-16T09:59:37,278 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37877 {}] master.HMaster(4169): Remote procedure done, pid=104 2024-11-16T09:59:37,280 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=104, resume processing ppid=103 2024-11-16T09:59:37,280 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=104, ppid=103, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.0320 sec 2024-11-16T09:59:37,282 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=103, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=103, table=TestAcidGuarantees in 2.0360 sec 2024-11-16T09:59:37,392 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37877 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=103 2024-11-16T09:59:37,393 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-16T09:59:37,393 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37877 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-16T09:59:37,395 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37877 {}] procedure2.ProcedureExecutor(1139): Stored pid=105, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=105, table=TestAcidGuarantees 2024-11-16T09:59:37,395 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37877 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=105 2024-11-16T09:59:37,396 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=105, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=105, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-16T09:59:37,397 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=105, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=105, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-16T09:59:37,397 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=106, ppid=105, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-16T09:59:37,425 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] regionserver.HRegion(8855): Flush requested on ffe1934c280093c181c2b062a05e0908 2024-11-16T09:59:37,425 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-16T09:59:37,426 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing ffe1934c280093c181c2b062a05e0908 3/3 column families, dataSize=80.51 KB heapSize=211.69 KB 2024-11-16T09:59:37,462 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241116822785b22ac548a6831fc5b09e62d398_ffe1934c280093c181c2b062a05e0908 is 50, key is test_row_0/A:col10/1731751176268/Put/seqid=0 2024-11-16T09:59:37,466 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37591 is added to blk_1073742219_1395 (size=12304) 2024-11-16T09:59:37,495 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffe1934c280093c181c2b062a05e0908, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T09:59:37,495 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffe1934c280093c181c2b062a05e0908, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T09:59:37,495 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43332 deadline: 1731751237484, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffe1934c280093c181c2b062a05e0908, server=037c58a19ddb,37267,1731751077844 2024-11-16T09:59:37,495 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43416 deadline: 1731751237485, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffe1934c280093c181c2b062a05e0908, server=037c58a19ddb,37267,1731751077844 2024-11-16T09:59:37,496 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731751162662.ffe1934c280093c181c2b062a05e0908., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 , the old value is region=TestAcidGuarantees,,1731751162662.ffe1934c280093c181c2b062a05e0908., hostname=037c58a19ddb,37267,1731751077844, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffe1934c280093c181c2b062a05e0908, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:59:37,496 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731751162662.ffe1934c280093c181c2b062a05e0908., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 , the old value is region=TestAcidGuarantees,,1731751162662.ffe1934c280093c181c2b062a05e0908., hostname=037c58a19ddb,37267,1731751077844, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffe1934c280093c181c2b062a05e0908, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:59:37,496 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731751162662.ffe1934c280093c181c2b062a05e0908., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffe1934c280093c181c2b062a05e0908, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:59:37,496 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731751162662.ffe1934c280093c181c2b062a05e0908., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffe1934c280093c181c2b062a05e0908, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:59:37,496 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731751162662.ffe1934c280093c181c2b062a05e0908., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 because the exception is null or not the one we care about 2024-11-16T09:59:37,496 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731751162662.ffe1934c280093c181c2b062a05e0908., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 because the exception is null or not the one we care about 2024-11-16T09:59:37,503 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37877 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=105 2024-11-16T09:59:37,505 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffe1934c280093c181c2b062a05e0908, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T09:59:37,505 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43402 deadline: 1731751237494, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffe1934c280093c181c2b062a05e0908, server=037c58a19ddb,37267,1731751077844 2024-11-16T09:59:37,505 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffe1934c280093c181c2b062a05e0908, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T09:59:37,505 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731751162662.ffe1934c280093c181c2b062a05e0908., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 , the old value is region=TestAcidGuarantees,,1731751162662.ffe1934c280093c181c2b062a05e0908., hostname=037c58a19ddb,37267,1731751077844, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffe1934c280093c181c2b062a05e0908, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:59:37,506 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731751162662.ffe1934c280093c181c2b062a05e0908., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffe1934c280093c181c2b062a05e0908, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:59:37,506 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43362 deadline: 1731751237495, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffe1934c280093c181c2b062a05e0908, server=037c58a19ddb,37267,1731751077844 2024-11-16T09:59:37,506 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731751162662.ffe1934c280093c181c2b062a05e0908., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 because the exception is null or not the one we care about 2024-11-16T09:59:37,506 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffe1934c280093c181c2b062a05e0908, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T09:59:37,506 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731751162662.ffe1934c280093c181c2b062a05e0908., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 , the old value is region=TestAcidGuarantees,,1731751162662.ffe1934c280093c181c2b062a05e0908., hostname=037c58a19ddb,37267,1731751077844, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffe1934c280093c181c2b062a05e0908, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:59:37,506 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731751162662.ffe1934c280093c181c2b062a05e0908., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffe1934c280093c181c2b062a05e0908, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:59:37,506 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731751162662.ffe1934c280093c181c2b062a05e0908., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 because the exception is null or not the one we care about 2024-11-16T09:59:37,506 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43418 deadline: 1731751237496, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffe1934c280093c181c2b062a05e0908, server=037c58a19ddb,37267,1731751077844 2024-11-16T09:59:37,506 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731751162662.ffe1934c280093c181c2b062a05e0908., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 , the old value is region=TestAcidGuarantees,,1731751162662.ffe1934c280093c181c2b062a05e0908., hostname=037c58a19ddb,37267,1731751077844, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffe1934c280093c181c2b062a05e0908, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:59:37,506 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731751162662.ffe1934c280093c181c2b062a05e0908., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffe1934c280093c181c2b062a05e0908, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:59:37,506 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731751162662.ffe1934c280093c181c2b062a05e0908., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 because the exception is null or not the one we care about 2024-11-16T09:59:37,549 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=37267 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=106 2024-11-16T09:59:37,549 DEBUG [RS_FLUSH_OPERATIONS-regionserver/037c58a19ddb:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731751162662.ffe1934c280093c181c2b062a05e0908. 2024-11-16T09:59:37,549 DEBUG [RS_FLUSH_OPERATIONS-regionserver/037c58a19ddb:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731751162662.ffe1934c280093c181c2b062a05e0908. as already flushing 2024-11-16T09:59:37,550 DEBUG [RS_FLUSH_OPERATIONS-regionserver/037c58a19ddb:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731751162662.ffe1934c280093c181c2b062a05e0908. 2024-11-16T09:59:37,550 ERROR [RS_FLUSH_OPERATIONS-regionserver/037c58a19ddb:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] handler.RSProcedureHandler(58): pid=106 java.io.IOException: Unable to complete flush {ENCODED => ffe1934c280093c181c2b062a05e0908, NAME => 'TestAcidGuarantees,,1731751162662.ffe1934c280093c181c2b062a05e0908.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T09:59:37,550 DEBUG [RS_FLUSH_OPERATIONS-regionserver/037c58a19ddb:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=106 java.io.IOException: Unable to complete flush {ENCODED => ffe1934c280093c181c2b062a05e0908, NAME => 'TestAcidGuarantees,,1731751162662.ffe1934c280093c181c2b062a05e0908.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T09:59:37,550 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37877 {}] master.HMaster(4177): Remote procedure failed, pid=106 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ffe1934c280093c181c2b062a05e0908, NAME => 'TestAcidGuarantees,,1731751162662.ffe1934c280093c181c2b062a05e0908.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ffe1934c280093c181c2b062a05e0908, NAME => 'TestAcidGuarantees,,1731751162662.ffe1934c280093c181c2b062a05e0908.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T09:59:37,612 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffe1934c280093c181c2b062a05e0908, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T09:59:37,612 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43416 deadline: 1731751237603, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffe1934c280093c181c2b062a05e0908, server=037c58a19ddb,37267,1731751077844 2024-11-16T09:59:37,612 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffe1934c280093c181c2b062a05e0908, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T09:59:37,612 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43332 deadline: 1731751237603, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffe1934c280093c181c2b062a05e0908, server=037c58a19ddb,37267,1731751077844 2024-11-16T09:59:37,613 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731751162662.ffe1934c280093c181c2b062a05e0908., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 , the old value is region=TestAcidGuarantees,,1731751162662.ffe1934c280093c181c2b062a05e0908., hostname=037c58a19ddb,37267,1731751077844, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffe1934c280093c181c2b062a05e0908, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:59:37,613 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731751162662.ffe1934c280093c181c2b062a05e0908., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffe1934c280093c181c2b062a05e0908, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:59:37,613 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731751162662.ffe1934c280093c181c2b062a05e0908., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 because the exception is null or not the one we care about 2024-11-16T09:59:37,613 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731751162662.ffe1934c280093c181c2b062a05e0908., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 , the old value is region=TestAcidGuarantees,,1731751162662.ffe1934c280093c181c2b062a05e0908., hostname=037c58a19ddb,37267,1731751077844, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffe1934c280093c181c2b062a05e0908, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:59:37,613 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731751162662.ffe1934c280093c181c2b062a05e0908., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffe1934c280093c181c2b062a05e0908, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:59:37,613 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731751162662.ffe1934c280093c181c2b062a05e0908., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 because the exception is null or not the one we care about 2024-11-16T09:59:37,617 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffe1934c280093c181c2b062a05e0908, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T09:59:37,617 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] ipc.CallRunner(138): callId: 71 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43402 deadline: 1731751237612, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffe1934c280093c181c2b062a05e0908, server=037c58a19ddb,37267,1731751077844 2024-11-16T09:59:37,618 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffe1934c280093c181c2b062a05e0908, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T09:59:37,618 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731751162662.ffe1934c280093c181c2b062a05e0908., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 , the old value is region=TestAcidGuarantees,,1731751162662.ffe1934c280093c181c2b062a05e0908., hostname=037c58a19ddb,37267,1731751077844, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffe1934c280093c181c2b062a05e0908, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:59:37,618 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731751162662.ffe1934c280093c181c2b062a05e0908., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffe1934c280093c181c2b062a05e0908, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:59:37,618 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43362 deadline: 1731751237612, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffe1934c280093c181c2b062a05e0908, server=037c58a19ddb,37267,1731751077844 2024-11-16T09:59:37,618 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731751162662.ffe1934c280093c181c2b062a05e0908., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 because the exception is null or not the one we care about 2024-11-16T09:59:37,618 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731751162662.ffe1934c280093c181c2b062a05e0908., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 , the old value is region=TestAcidGuarantees,,1731751162662.ffe1934c280093c181c2b062a05e0908., hostname=037c58a19ddb,37267,1731751077844, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffe1934c280093c181c2b062a05e0908, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:59:37,618 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffe1934c280093c181c2b062a05e0908, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T09:59:37,618 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731751162662.ffe1934c280093c181c2b062a05e0908., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffe1934c280093c181c2b062a05e0908, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:59:37,618 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731751162662.ffe1934c280093c181c2b062a05e0908., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 because the exception is null or not the one we care about 2024-11-16T09:59:37,618 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43418 deadline: 1731751237613, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffe1934c280093c181c2b062a05e0908, server=037c58a19ddb,37267,1731751077844 2024-11-16T09:59:37,618 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731751162662.ffe1934c280093c181c2b062a05e0908., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 , the old value is region=TestAcidGuarantees,,1731751162662.ffe1934c280093c181c2b062a05e0908., hostname=037c58a19ddb,37267,1731751077844, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffe1934c280093c181c2b062a05e0908, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:59:37,618 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731751162662.ffe1934c280093c181c2b062a05e0908., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffe1934c280093c181c2b062a05e0908, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:59:37,618 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731751162662.ffe1934c280093c181c2b062a05e0908., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 because the exception is null or not the one we care about 2024-11-16T09:59:37,702 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=37267 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=106 2024-11-16T09:59:37,702 DEBUG [RS_FLUSH_OPERATIONS-regionserver/037c58a19ddb:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731751162662.ffe1934c280093c181c2b062a05e0908. 2024-11-16T09:59:37,702 DEBUG [RS_FLUSH_OPERATIONS-regionserver/037c58a19ddb:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731751162662.ffe1934c280093c181c2b062a05e0908. as already flushing 2024-11-16T09:59:37,702 DEBUG [RS_FLUSH_OPERATIONS-regionserver/037c58a19ddb:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731751162662.ffe1934c280093c181c2b062a05e0908. 2024-11-16T09:59:37,703 ERROR [RS_FLUSH_OPERATIONS-regionserver/037c58a19ddb:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] handler.RSProcedureHandler(58): pid=106 java.io.IOException: Unable to complete flush {ENCODED => ffe1934c280093c181c2b062a05e0908, NAME => 'TestAcidGuarantees,,1731751162662.ffe1934c280093c181c2b062a05e0908.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T09:59:37,703 DEBUG [RS_FLUSH_OPERATIONS-regionserver/037c58a19ddb:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=106 java.io.IOException: Unable to complete flush {ENCODED => ffe1934c280093c181c2b062a05e0908, NAME => 'TestAcidGuarantees,,1731751162662.ffe1934c280093c181c2b062a05e0908.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T09:59:37,703 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37877 {}] master.HMaster(4177): Remote procedure failed, pid=106 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ffe1934c280093c181c2b062a05e0908, NAME => 'TestAcidGuarantees,,1731751162662.ffe1934c280093c181c2b062a05e0908.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ffe1934c280093c181c2b062a05e0908, NAME => 'TestAcidGuarantees,,1731751162662.ffe1934c280093c181c2b062a05e0908.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T09:59:37,712 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37877 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=105 2024-11-16T09:59:37,826 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffe1934c280093c181c2b062a05e0908, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T09:59:37,826 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43416 deadline: 1731751237822, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffe1934c280093c181c2b062a05e0908, server=037c58a19ddb,37267,1731751077844 2024-11-16T09:59:37,827 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffe1934c280093c181c2b062a05e0908, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T09:59:37,827 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43332 deadline: 1731751237822, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffe1934c280093c181c2b062a05e0908, server=037c58a19ddb,37267,1731751077844 2024-11-16T09:59:37,827 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731751162662.ffe1934c280093c181c2b062a05e0908., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 , the old value is region=TestAcidGuarantees,,1731751162662.ffe1934c280093c181c2b062a05e0908., hostname=037c58a19ddb,37267,1731751077844, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffe1934c280093c181c2b062a05e0908, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:59:37,827 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731751162662.ffe1934c280093c181c2b062a05e0908., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffe1934c280093c181c2b062a05e0908, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:59:37,827 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731751162662.ffe1934c280093c181c2b062a05e0908., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 because the exception is null or not the one we care about 2024-11-16T09:59:37,827 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffe1934c280093c181c2b062a05e0908, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T09:59:37,827 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731751162662.ffe1934c280093c181c2b062a05e0908., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 , the old value is region=TestAcidGuarantees,,1731751162662.ffe1934c280093c181c2b062a05e0908., hostname=037c58a19ddb,37267,1731751077844, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffe1934c280093c181c2b062a05e0908, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:59:37,827 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731751162662.ffe1934c280093c181c2b062a05e0908., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffe1934c280093c181c2b062a05e0908, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:59:37,827 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] ipc.CallRunner(138): callId: 72 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43402 deadline: 1731751237822, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffe1934c280093c181c2b062a05e0908, server=037c58a19ddb,37267,1731751077844 2024-11-16T09:59:37,827 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731751162662.ffe1934c280093c181c2b062a05e0908., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 because the exception is null or not the one we care about 2024-11-16T09:59:37,827 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731751162662.ffe1934c280093c181c2b062a05e0908., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 , the old value is region=TestAcidGuarantees,,1731751162662.ffe1934c280093c181c2b062a05e0908., hostname=037c58a19ddb,37267,1731751077844, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffe1934c280093c181c2b062a05e0908, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:59:37,827 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffe1934c280093c181c2b062a05e0908, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T09:59:37,827 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731751162662.ffe1934c280093c181c2b062a05e0908., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffe1934c280093c181c2b062a05e0908, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:59:37,827 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731751162662.ffe1934c280093c181c2b062a05e0908., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 because the exception is null or not the one we care about 2024-11-16T09:59:37,827 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43362 deadline: 1731751237823, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffe1934c280093c181c2b062a05e0908, server=037c58a19ddb,37267,1731751077844 2024-11-16T09:59:37,828 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731751162662.ffe1934c280093c181c2b062a05e0908., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 , the old value is region=TestAcidGuarantees,,1731751162662.ffe1934c280093c181c2b062a05e0908., hostname=037c58a19ddb,37267,1731751077844, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffe1934c280093c181c2b062a05e0908, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:59:37,828 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731751162662.ffe1934c280093c181c2b062a05e0908., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffe1934c280093c181c2b062a05e0908, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:59:37,828 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffe1934c280093c181c2b062a05e0908, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T09:59:37,828 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731751162662.ffe1934c280093c181c2b062a05e0908., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 because the exception is null or not the one we care about 2024-11-16T09:59:37,828 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43418 deadline: 1731751237823, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffe1934c280093c181c2b062a05e0908, server=037c58a19ddb,37267,1731751077844 2024-11-16T09:59:37,828 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731751162662.ffe1934c280093c181c2b062a05e0908., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 , the old value is region=TestAcidGuarantees,,1731751162662.ffe1934c280093c181c2b062a05e0908., hostname=037c58a19ddb,37267,1731751077844, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffe1934c280093c181c2b062a05e0908, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:59:37,828 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731751162662.ffe1934c280093c181c2b062a05e0908., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffe1934c280093c181c2b062a05e0908, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:59:37,828 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731751162662.ffe1934c280093c181c2b062a05e0908., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 because the exception is null or not the one we care about 2024-11-16T09:59:37,855 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=37267 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=106 2024-11-16T09:59:37,855 DEBUG [RS_FLUSH_OPERATIONS-regionserver/037c58a19ddb:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731751162662.ffe1934c280093c181c2b062a05e0908. 2024-11-16T09:59:37,855 DEBUG [RS_FLUSH_OPERATIONS-regionserver/037c58a19ddb:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731751162662.ffe1934c280093c181c2b062a05e0908. as already flushing 2024-11-16T09:59:37,855 DEBUG [RS_FLUSH_OPERATIONS-regionserver/037c58a19ddb:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731751162662.ffe1934c280093c181c2b062a05e0908. 2024-11-16T09:59:37,855 ERROR [RS_FLUSH_OPERATIONS-regionserver/037c58a19ddb:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] handler.RSProcedureHandler(58): pid=106 java.io.IOException: Unable to complete flush {ENCODED => ffe1934c280093c181c2b062a05e0908, NAME => 'TestAcidGuarantees,,1731751162662.ffe1934c280093c181c2b062a05e0908.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T09:59:37,856 DEBUG [RS_FLUSH_OPERATIONS-regionserver/037c58a19ddb:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=106 java.io.IOException: Unable to complete flush {ENCODED => ffe1934c280093c181c2b062a05e0908, NAME => 'TestAcidGuarantees,,1731751162662.ffe1934c280093c181c2b062a05e0908.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T09:59:37,856 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37877 {}] master.HMaster(4177): Remote procedure failed, pid=106 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ffe1934c280093c181c2b062a05e0908, NAME => 'TestAcidGuarantees,,1731751162662.ffe1934c280093c181c2b062a05e0908.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ffe1934c280093c181c2b062a05e0908, NAME => 'TestAcidGuarantees,,1731751162662.ffe1934c280093c181c2b062a05e0908.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T09:59:37,866 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T09:59:37,870 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241116822785b22ac548a6831fc5b09e62d398_ffe1934c280093c181c2b062a05e0908 to hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241116822785b22ac548a6831fc5b09e62d398_ffe1934c280093c181c2b062a05e0908 2024-11-16T09:59:37,871 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/ffe1934c280093c181c2b062a05e0908/.tmp/A/496b53efeb1648a1ae19d0a8097b6c2b, store: [table=TestAcidGuarantees family=A region=ffe1934c280093c181c2b062a05e0908] 2024-11-16T09:59:37,872 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/ffe1934c280093c181c2b062a05e0908/.tmp/A/496b53efeb1648a1ae19d0a8097b6c2b is 175, key is test_row_0/A:col10/1731751176268/Put/seqid=0 2024-11-16T09:59:37,875 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37591 is added to blk_1073742220_1396 (size=31105) 2024-11-16T09:59:38,008 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=37267 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=106 2024-11-16T09:59:38,008 DEBUG [RS_FLUSH_OPERATIONS-regionserver/037c58a19ddb:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731751162662.ffe1934c280093c181c2b062a05e0908. 2024-11-16T09:59:38,009 DEBUG [RS_FLUSH_OPERATIONS-regionserver/037c58a19ddb:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731751162662.ffe1934c280093c181c2b062a05e0908. as already flushing 2024-11-16T09:59:38,009 DEBUG [RS_FLUSH_OPERATIONS-regionserver/037c58a19ddb:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731751162662.ffe1934c280093c181c2b062a05e0908. 2024-11-16T09:59:38,009 ERROR [RS_FLUSH_OPERATIONS-regionserver/037c58a19ddb:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] handler.RSProcedureHandler(58): pid=106 java.io.IOException: Unable to complete flush {ENCODED => ffe1934c280093c181c2b062a05e0908, NAME => 'TestAcidGuarantees,,1731751162662.ffe1934c280093c181c2b062a05e0908.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T09:59:38,009 DEBUG [RS_FLUSH_OPERATIONS-regionserver/037c58a19ddb:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=106 java.io.IOException: Unable to complete flush {ENCODED => ffe1934c280093c181c2b062a05e0908, NAME => 'TestAcidGuarantees,,1731751162662.ffe1934c280093c181c2b062a05e0908.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T09:59:38,009 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37877 {}] master.HMaster(4177): Remote procedure failed, pid=106 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ffe1934c280093c181c2b062a05e0908, NAME => 'TestAcidGuarantees,,1731751162662.ffe1934c280093c181c2b062a05e0908.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ffe1934c280093c181c2b062a05e0908, NAME => 'TestAcidGuarantees,,1731751162662.ffe1934c280093c181c2b062a05e0908.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T09:59:38,022 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37877 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=105 2024-11-16T09:59:38,135 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffe1934c280093c181c2b062a05e0908, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T09:59:38,135 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43416 deadline: 1731751238132, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffe1934c280093c181c2b062a05e0908, server=037c58a19ddb,37267,1731751077844 2024-11-16T09:59:38,135 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffe1934c280093c181c2b062a05e0908, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T09:59:38,135 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731751162662.ffe1934c280093c181c2b062a05e0908., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 , the old value is region=TestAcidGuarantees,,1731751162662.ffe1934c280093c181c2b062a05e0908., hostname=037c58a19ddb,37267,1731751077844, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffe1934c280093c181c2b062a05e0908, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:59:38,135 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731751162662.ffe1934c280093c181c2b062a05e0908., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffe1934c280093c181c2b062a05e0908, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:59:38,135 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731751162662.ffe1934c280093c181c2b062a05e0908., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 because the exception is null or not the one we care about 2024-11-16T09:59:38,135 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43332 deadline: 1731751238132, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffe1934c280093c181c2b062a05e0908, server=037c58a19ddb,37267,1731751077844 2024-11-16T09:59:38,136 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731751162662.ffe1934c280093c181c2b062a05e0908., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 , the old value is region=TestAcidGuarantees,,1731751162662.ffe1934c280093c181c2b062a05e0908., hostname=037c58a19ddb,37267,1731751077844, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffe1934c280093c181c2b062a05e0908, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:59:38,136 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffe1934c280093c181c2b062a05e0908, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T09:59:38,136 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731751162662.ffe1934c280093c181c2b062a05e0908., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffe1934c280093c181c2b062a05e0908, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:59:38,136 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731751162662.ffe1934c280093c181c2b062a05e0908., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 because the exception is null or not the one we care about 2024-11-16T09:59:38,136 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43418 deadline: 1731751238133, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffe1934c280093c181c2b062a05e0908, server=037c58a19ddb,37267,1731751077844 2024-11-16T09:59:38,136 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731751162662.ffe1934c280093c181c2b062a05e0908., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 , the old value is region=TestAcidGuarantees,,1731751162662.ffe1934c280093c181c2b062a05e0908., hostname=037c58a19ddb,37267,1731751077844, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffe1934c280093c181c2b062a05e0908, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:59:38,136 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731751162662.ffe1934c280093c181c2b062a05e0908., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffe1934c280093c181c2b062a05e0908, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:59:38,136 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731751162662.ffe1934c280093c181c2b062a05e0908., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 because the exception is null or not the one we care about 2024-11-16T09:59:38,141 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffe1934c280093c181c2b062a05e0908, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T09:59:38,141 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43362 deadline: 1731751238133, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffe1934c280093c181c2b062a05e0908, server=037c58a19ddb,37267,1731751077844 2024-11-16T09:59:38,141 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731751162662.ffe1934c280093c181c2b062a05e0908., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 , the old value is region=TestAcidGuarantees,,1731751162662.ffe1934c280093c181c2b062a05e0908., hostname=037c58a19ddb,37267,1731751077844, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffe1934c280093c181c2b062a05e0908, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:59:38,141 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffe1934c280093c181c2b062a05e0908, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T09:59:38,141 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731751162662.ffe1934c280093c181c2b062a05e0908., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffe1934c280093c181c2b062a05e0908, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:59:38,141 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731751162662.ffe1934c280093c181c2b062a05e0908., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 because the exception is null or not the one we care about 2024-11-16T09:59:38,141 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] ipc.CallRunner(138): callId: 73 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43402 deadline: 1731751238133, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffe1934c280093c181c2b062a05e0908, server=037c58a19ddb,37267,1731751077844 2024-11-16T09:59:38,141 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731751162662.ffe1934c280093c181c2b062a05e0908., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 , the old value is region=TestAcidGuarantees,,1731751162662.ffe1934c280093c181c2b062a05e0908., hostname=037c58a19ddb,37267,1731751077844, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffe1934c280093c181c2b062a05e0908, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:59:38,141 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731751162662.ffe1934c280093c181c2b062a05e0908., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffe1934c280093c181c2b062a05e0908, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:59:38,141 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731751162662.ffe1934c280093c181c2b062a05e0908., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 because the exception is null or not the one we care about 2024-11-16T09:59:38,161 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=37267 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=106 2024-11-16T09:59:38,161 DEBUG [RS_FLUSH_OPERATIONS-regionserver/037c58a19ddb:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731751162662.ffe1934c280093c181c2b062a05e0908. 2024-11-16T09:59:38,162 DEBUG [RS_FLUSH_OPERATIONS-regionserver/037c58a19ddb:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731751162662.ffe1934c280093c181c2b062a05e0908. as already flushing 2024-11-16T09:59:38,162 DEBUG [RS_FLUSH_OPERATIONS-regionserver/037c58a19ddb:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731751162662.ffe1934c280093c181c2b062a05e0908. 2024-11-16T09:59:38,162 ERROR [RS_FLUSH_OPERATIONS-regionserver/037c58a19ddb:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] handler.RSProcedureHandler(58): pid=106 java.io.IOException: Unable to complete flush {ENCODED => ffe1934c280093c181c2b062a05e0908, NAME => 'TestAcidGuarantees,,1731751162662.ffe1934c280093c181c2b062a05e0908.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T09:59:38,162 DEBUG [RS_FLUSH_OPERATIONS-regionserver/037c58a19ddb:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=106 java.io.IOException: Unable to complete flush {ENCODED => ffe1934c280093c181c2b062a05e0908, NAME => 'TestAcidGuarantees,,1731751162662.ffe1934c280093c181c2b062a05e0908.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T09:59:38,163 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37877 {}] master.HMaster(4177): Remote procedure failed, pid=106 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ffe1934c280093c181c2b062a05e0908, NAME => 'TestAcidGuarantees,,1731751162662.ffe1934c280093c181c2b062a05e0908.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ffe1934c280093c181c2b062a05e0908, NAME => 'TestAcidGuarantees,,1731751162662.ffe1934c280093c181c2b062a05e0908.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T09:59:38,275 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=187, memsize=26.8 K, hasBloomFilter=true, into tmp file hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/ffe1934c280093c181c2b062a05e0908/.tmp/A/496b53efeb1648a1ae19d0a8097b6c2b 2024-11-16T09:59:38,282 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/ffe1934c280093c181c2b062a05e0908/.tmp/B/7ae7f0d8dc8547829a327b15ba583cf5 is 50, key is test_row_0/B:col10/1731751176268/Put/seqid=0 2024-11-16T09:59:38,287 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37591 is added to blk_1073742221_1397 (size=12151) 2024-11-16T09:59:38,314 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=37267 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=106 2024-11-16T09:59:38,315 DEBUG [RS_FLUSH_OPERATIONS-regionserver/037c58a19ddb:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731751162662.ffe1934c280093c181c2b062a05e0908. 2024-11-16T09:59:38,315 DEBUG [RS_FLUSH_OPERATIONS-regionserver/037c58a19ddb:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731751162662.ffe1934c280093c181c2b062a05e0908. as already flushing 2024-11-16T09:59:38,315 DEBUG [RS_FLUSH_OPERATIONS-regionserver/037c58a19ddb:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731751162662.ffe1934c280093c181c2b062a05e0908. 2024-11-16T09:59:38,315 ERROR [RS_FLUSH_OPERATIONS-regionserver/037c58a19ddb:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] handler.RSProcedureHandler(58): pid=106 java.io.IOException: Unable to complete flush {ENCODED => ffe1934c280093c181c2b062a05e0908, NAME => 'TestAcidGuarantees,,1731751162662.ffe1934c280093c181c2b062a05e0908.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T09:59:38,315 DEBUG [RS_FLUSH_OPERATIONS-regionserver/037c58a19ddb:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=106 java.io.IOException: Unable to complete flush {ENCODED => ffe1934c280093c181c2b062a05e0908, NAME => 'TestAcidGuarantees,,1731751162662.ffe1934c280093c181c2b062a05e0908.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T09:59:38,316 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37877 {}] master.HMaster(4177): Remote procedure failed, pid=106 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ffe1934c280093c181c2b062a05e0908, NAME => 'TestAcidGuarantees,,1731751162662.ffe1934c280093c181c2b062a05e0908.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ffe1934c280093c181c2b062a05e0908, NAME => 'TestAcidGuarantees,,1731751162662.ffe1934c280093c181c2b062a05e0908.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T09:59:38,468 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=37267 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=106 2024-11-16T09:59:38,468 DEBUG [RS_FLUSH_OPERATIONS-regionserver/037c58a19ddb:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731751162662.ffe1934c280093c181c2b062a05e0908. 2024-11-16T09:59:38,468 DEBUG [RS_FLUSH_OPERATIONS-regionserver/037c58a19ddb:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731751162662.ffe1934c280093c181c2b062a05e0908. as already flushing 2024-11-16T09:59:38,468 DEBUG [RS_FLUSH_OPERATIONS-regionserver/037c58a19ddb:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731751162662.ffe1934c280093c181c2b062a05e0908. 2024-11-16T09:59:38,468 ERROR [RS_FLUSH_OPERATIONS-regionserver/037c58a19ddb:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] handler.RSProcedureHandler(58): pid=106 java.io.IOException: Unable to complete flush {ENCODED => ffe1934c280093c181c2b062a05e0908, NAME => 'TestAcidGuarantees,,1731751162662.ffe1934c280093c181c2b062a05e0908.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T09:59:38,468 DEBUG [RS_FLUSH_OPERATIONS-regionserver/037c58a19ddb:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=106 java.io.IOException: Unable to complete flush {ENCODED => ffe1934c280093c181c2b062a05e0908, NAME => 'TestAcidGuarantees,,1731751162662.ffe1934c280093c181c2b062a05e0908.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T09:59:38,469 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37877 {}] master.HMaster(4177): Remote procedure failed, pid=106 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ffe1934c280093c181c2b062a05e0908, NAME => 'TestAcidGuarantees,,1731751162662.ffe1934c280093c181c2b062a05e0908.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ffe1934c280093c181c2b062a05e0908, NAME => 'TestAcidGuarantees,,1731751162662.ffe1934c280093c181c2b062a05e0908.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T09:59:38,533 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37877 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=105 2024-11-16T09:59:38,621 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=37267 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=106 2024-11-16T09:59:38,621 DEBUG [RS_FLUSH_OPERATIONS-regionserver/037c58a19ddb:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731751162662.ffe1934c280093c181c2b062a05e0908. 2024-11-16T09:59:38,621 DEBUG [RS_FLUSH_OPERATIONS-regionserver/037c58a19ddb:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731751162662.ffe1934c280093c181c2b062a05e0908. as already flushing 2024-11-16T09:59:38,621 DEBUG [RS_FLUSH_OPERATIONS-regionserver/037c58a19ddb:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731751162662.ffe1934c280093c181c2b062a05e0908. 2024-11-16T09:59:38,621 ERROR [RS_FLUSH_OPERATIONS-regionserver/037c58a19ddb:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] handler.RSProcedureHandler(58): pid=106 java.io.IOException: Unable to complete flush {ENCODED => ffe1934c280093c181c2b062a05e0908, NAME => 'TestAcidGuarantees,,1731751162662.ffe1934c280093c181c2b062a05e0908.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T09:59:38,621 DEBUG [RS_FLUSH_OPERATIONS-regionserver/037c58a19ddb:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=106 java.io.IOException: Unable to complete flush {ENCODED => ffe1934c280093c181c2b062a05e0908, NAME => 'TestAcidGuarantees,,1731751162662.ffe1934c280093c181c2b062a05e0908.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T09:59:38,622 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37877 {}] master.HMaster(4177): Remote procedure failed, pid=106 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ffe1934c280093c181c2b062a05e0908, NAME => 'TestAcidGuarantees,,1731751162662.ffe1934c280093c181c2b062a05e0908.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ffe1934c280093c181c2b062a05e0908, NAME => 'TestAcidGuarantees,,1731751162662.ffe1934c280093c181c2b062a05e0908.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T09:59:38,650 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffe1934c280093c181c2b062a05e0908, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T09:59:38,650 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] ipc.CallRunner(138): callId: 71 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43416 deadline: 1731751238642, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffe1934c280093c181c2b062a05e0908, server=037c58a19ddb,37267,1731751077844 2024-11-16T09:59:38,651 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731751162662.ffe1934c280093c181c2b062a05e0908., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 , the old value is region=TestAcidGuarantees,,1731751162662.ffe1934c280093c181c2b062a05e0908., hostname=037c58a19ddb,37267,1731751077844, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffe1934c280093c181c2b062a05e0908, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:59:38,650 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffe1934c280093c181c2b062a05e0908, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T09:59:38,651 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731751162662.ffe1934c280093c181c2b062a05e0908., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffe1934c280093c181c2b062a05e0908, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:59:38,651 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731751162662.ffe1934c280093c181c2b062a05e0908., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 because the exception is null or not the one we care about 2024-11-16T09:59:38,651 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43332 deadline: 1731751238643, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffe1934c280093c181c2b062a05e0908, server=037c58a19ddb,37267,1731751077844 2024-11-16T09:59:38,651 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731751162662.ffe1934c280093c181c2b062a05e0908., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 , the old value is region=TestAcidGuarantees,,1731751162662.ffe1934c280093c181c2b062a05e0908., hostname=037c58a19ddb,37267,1731751077844, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffe1934c280093c181c2b062a05e0908, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:59:38,651 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731751162662.ffe1934c280093c181c2b062a05e0908., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffe1934c280093c181c2b062a05e0908, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:59:38,651 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffe1934c280093c181c2b062a05e0908, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T09:59:38,651 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731751162662.ffe1934c280093c181c2b062a05e0908., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 because the exception is null or not the one we care about 2024-11-16T09:59:38,651 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43418 deadline: 1731751238643, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffe1934c280093c181c2b062a05e0908, server=037c58a19ddb,37267,1731751077844 2024-11-16T09:59:38,651 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731751162662.ffe1934c280093c181c2b062a05e0908., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 , the old value is region=TestAcidGuarantees,,1731751162662.ffe1934c280093c181c2b062a05e0908., hostname=037c58a19ddb,37267,1731751077844, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffe1934c280093c181c2b062a05e0908, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:59:38,651 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731751162662.ffe1934c280093c181c2b062a05e0908., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffe1934c280093c181c2b062a05e0908, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:59:38,651 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731751162662.ffe1934c280093c181c2b062a05e0908., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 because the exception is null or not the one we care about 2024-11-16T09:59:38,663 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffe1934c280093c181c2b062a05e0908, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T09:59:38,663 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43362 deadline: 1731751238653, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffe1934c280093c181c2b062a05e0908, server=037c58a19ddb,37267,1731751077844 2024-11-16T09:59:38,663 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731751162662.ffe1934c280093c181c2b062a05e0908., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 , the old value is region=TestAcidGuarantees,,1731751162662.ffe1934c280093c181c2b062a05e0908., hostname=037c58a19ddb,37267,1731751077844, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffe1934c280093c181c2b062a05e0908, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:59:38,663 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731751162662.ffe1934c280093c181c2b062a05e0908., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffe1934c280093c181c2b062a05e0908, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:59:38,663 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffe1934c280093c181c2b062a05e0908, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T09:59:38,663 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731751162662.ffe1934c280093c181c2b062a05e0908., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 because the exception is null or not the one we care about 2024-11-16T09:59:38,663 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] ipc.CallRunner(138): callId: 74 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43402 deadline: 1731751238653, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffe1934c280093c181c2b062a05e0908, server=037c58a19ddb,37267,1731751077844 2024-11-16T09:59:38,664 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731751162662.ffe1934c280093c181c2b062a05e0908., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 , the old value is region=TestAcidGuarantees,,1731751162662.ffe1934c280093c181c2b062a05e0908., hostname=037c58a19ddb,37267,1731751077844, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffe1934c280093c181c2b062a05e0908, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:59:38,664 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731751162662.ffe1934c280093c181c2b062a05e0908., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffe1934c280093c181c2b062a05e0908, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:59:38,664 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731751162662.ffe1934c280093c181c2b062a05e0908., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 because the exception is null or not the one we care about 2024-11-16T09:59:38,688 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=187 (bloomFilter=true), to=hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/ffe1934c280093c181c2b062a05e0908/.tmp/B/7ae7f0d8dc8547829a327b15ba583cf5 2024-11-16T09:59:38,694 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/ffe1934c280093c181c2b062a05e0908/.tmp/C/20f06da3d4ca4b91a578717bd9dbaee0 is 50, key is test_row_0/C:col10/1731751176268/Put/seqid=0 2024-11-16T09:59:38,697 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37591 is added to blk_1073742222_1398 (size=12151) 2024-11-16T09:59:38,774 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=37267 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=106 2024-11-16T09:59:38,774 DEBUG [RS_FLUSH_OPERATIONS-regionserver/037c58a19ddb:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731751162662.ffe1934c280093c181c2b062a05e0908. 2024-11-16T09:59:38,774 DEBUG [RS_FLUSH_OPERATIONS-regionserver/037c58a19ddb:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731751162662.ffe1934c280093c181c2b062a05e0908. as already flushing 2024-11-16T09:59:38,774 DEBUG [RS_FLUSH_OPERATIONS-regionserver/037c58a19ddb:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731751162662.ffe1934c280093c181c2b062a05e0908. 2024-11-16T09:59:38,774 ERROR [RS_FLUSH_OPERATIONS-regionserver/037c58a19ddb:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] handler.RSProcedureHandler(58): pid=106 java.io.IOException: Unable to complete flush {ENCODED => ffe1934c280093c181c2b062a05e0908, NAME => 'TestAcidGuarantees,,1731751162662.ffe1934c280093c181c2b062a05e0908.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T09:59:38,775 DEBUG [RS_FLUSH_OPERATIONS-regionserver/037c58a19ddb:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=106 java.io.IOException: Unable to complete flush {ENCODED => ffe1934c280093c181c2b062a05e0908, NAME => 'TestAcidGuarantees,,1731751162662.ffe1934c280093c181c2b062a05e0908.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T09:59:38,775 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37877 {}] master.HMaster(4177): Remote procedure failed, pid=106 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ffe1934c280093c181c2b062a05e0908, NAME => 'TestAcidGuarantees,,1731751162662.ffe1934c280093c181c2b062a05e0908.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ffe1934c280093c181c2b062a05e0908, NAME => 'TestAcidGuarantees,,1731751162662.ffe1934c280093c181c2b062a05e0908.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T09:59:38,927 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=37267 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=106 2024-11-16T09:59:38,927 DEBUG [RS_FLUSH_OPERATIONS-regionserver/037c58a19ddb:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731751162662.ffe1934c280093c181c2b062a05e0908. 2024-11-16T09:59:38,928 DEBUG [RS_FLUSH_OPERATIONS-regionserver/037c58a19ddb:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731751162662.ffe1934c280093c181c2b062a05e0908. as already flushing 2024-11-16T09:59:38,928 DEBUG [RS_FLUSH_OPERATIONS-regionserver/037c58a19ddb:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731751162662.ffe1934c280093c181c2b062a05e0908. 2024-11-16T09:59:38,928 ERROR [RS_FLUSH_OPERATIONS-regionserver/037c58a19ddb:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] handler.RSProcedureHandler(58): pid=106 java.io.IOException: Unable to complete flush {ENCODED => ffe1934c280093c181c2b062a05e0908, NAME => 'TestAcidGuarantees,,1731751162662.ffe1934c280093c181c2b062a05e0908.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T09:59:38,928 DEBUG [RS_FLUSH_OPERATIONS-regionserver/037c58a19ddb:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=106 java.io.IOException: Unable to complete flush {ENCODED => ffe1934c280093c181c2b062a05e0908, NAME => 'TestAcidGuarantees,,1731751162662.ffe1934c280093c181c2b062a05e0908.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T09:59:38,928 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37877 {}] master.HMaster(4177): Remote procedure failed, pid=106 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ffe1934c280093c181c2b062a05e0908, NAME => 'TestAcidGuarantees,,1731751162662.ffe1934c280093c181c2b062a05e0908.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ffe1934c280093c181c2b062a05e0908, NAME => 'TestAcidGuarantees,,1731751162662.ffe1934c280093c181c2b062a05e0908.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T09:59:39,080 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=37267 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=106 2024-11-16T09:59:39,081 DEBUG [RS_FLUSH_OPERATIONS-regionserver/037c58a19ddb:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731751162662.ffe1934c280093c181c2b062a05e0908. 2024-11-16T09:59:39,081 DEBUG [RS_FLUSH_OPERATIONS-regionserver/037c58a19ddb:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731751162662.ffe1934c280093c181c2b062a05e0908. as already flushing 2024-11-16T09:59:39,081 DEBUG [RS_FLUSH_OPERATIONS-regionserver/037c58a19ddb:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731751162662.ffe1934c280093c181c2b062a05e0908. 2024-11-16T09:59:39,081 ERROR [RS_FLUSH_OPERATIONS-regionserver/037c58a19ddb:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] handler.RSProcedureHandler(58): pid=106 java.io.IOException: Unable to complete flush {ENCODED => ffe1934c280093c181c2b062a05e0908, NAME => 'TestAcidGuarantees,,1731751162662.ffe1934c280093c181c2b062a05e0908.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T09:59:39,081 DEBUG [RS_FLUSH_OPERATIONS-regionserver/037c58a19ddb:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=106 java.io.IOException: Unable to complete flush {ENCODED => ffe1934c280093c181c2b062a05e0908, NAME => 'TestAcidGuarantees,,1731751162662.ffe1934c280093c181c2b062a05e0908.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T09:59:39,081 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37877 {}] master.HMaster(4177): Remote procedure failed, pid=106 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ffe1934c280093c181c2b062a05e0908, NAME => 'TestAcidGuarantees,,1731751162662.ffe1934c280093c181c2b062a05e0908.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ffe1934c280093c181c2b062a05e0908, NAME => 'TestAcidGuarantees,,1731751162662.ffe1934c280093c181c2b062a05e0908.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T09:59:39,098 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=187 (bloomFilter=true), to=hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/ffe1934c280093c181c2b062a05e0908/.tmp/C/20f06da3d4ca4b91a578717bd9dbaee0 2024-11-16T09:59:39,102 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/ffe1934c280093c181c2b062a05e0908/.tmp/A/496b53efeb1648a1ae19d0a8097b6c2b as hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/ffe1934c280093c181c2b062a05e0908/A/496b53efeb1648a1ae19d0a8097b6c2b 2024-11-16T09:59:39,105 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/ffe1934c280093c181c2b062a05e0908/A/496b53efeb1648a1ae19d0a8097b6c2b, entries=150, sequenceid=187, filesize=30.4 K 2024-11-16T09:59:39,106 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/ffe1934c280093c181c2b062a05e0908/.tmp/B/7ae7f0d8dc8547829a327b15ba583cf5 as hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/ffe1934c280093c181c2b062a05e0908/B/7ae7f0d8dc8547829a327b15ba583cf5 2024-11-16T09:59:39,109 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/ffe1934c280093c181c2b062a05e0908/B/7ae7f0d8dc8547829a327b15ba583cf5, entries=150, sequenceid=187, filesize=11.9 K 2024-11-16T09:59:39,110 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/ffe1934c280093c181c2b062a05e0908/.tmp/C/20f06da3d4ca4b91a578717bd9dbaee0 as hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/ffe1934c280093c181c2b062a05e0908/C/20f06da3d4ca4b91a578717bd9dbaee0 2024-11-16T09:59:39,113 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/ffe1934c280093c181c2b062a05e0908/C/20f06da3d4ca4b91a578717bd9dbaee0, entries=150, sequenceid=187, filesize=11.9 K 2024-11-16T09:59:39,114 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~80.51 KB/82440, heapSize ~211.64 KB/216720, currentSize=127.47 KB/130530 for ffe1934c280093c181c2b062a05e0908 in 1689ms, sequenceid=187, compaction requested=true 2024-11-16T09:59:39,114 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for ffe1934c280093c181c2b062a05e0908: 2024-11-16T09:59:39,114 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store ffe1934c280093c181c2b062a05e0908:A, priority=-2147483648, current under compaction store size is 1 2024-11-16T09:59:39,114 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-16T09:59:39,114 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store ffe1934c280093c181c2b062a05e0908:B, priority=-2147483648, current under compaction store size is 2 2024-11-16T09:59:39,114 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-16T09:59:39,114 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store ffe1934c280093c181c2b062a05e0908:C, priority=-2147483648, current under compaction store size is 3 2024-11-16T09:59:39,114 DEBUG [RS:0;037c58a19ddb:37267-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-16T09:59:39,115 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-16T09:59:39,114 DEBUG [RS:0;037c58a19ddb:37267-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-16T09:59:39,115 DEBUG [RS:0;037c58a19ddb:37267-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 93657 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-16T09:59:39,115 DEBUG [RS:0;037c58a19ddb:37267-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36795 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-16T09:59:39,115 DEBUG [RS:0;037c58a19ddb:37267-shortCompactions-0 {}] regionserver.HStore(1541): ffe1934c280093c181c2b062a05e0908/A is initiating minor compaction (all files) 2024-11-16T09:59:39,115 DEBUG [RS:0;037c58a19ddb:37267-longCompactions-0 {}] regionserver.HStore(1541): ffe1934c280093c181c2b062a05e0908/B is initiating minor compaction (all files) 2024-11-16T09:59:39,115 INFO [RS:0;037c58a19ddb:37267-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of ffe1934c280093c181c2b062a05e0908/A in TestAcidGuarantees,,1731751162662.ffe1934c280093c181c2b062a05e0908. 2024-11-16T09:59:39,115 INFO [RS:0;037c58a19ddb:37267-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of ffe1934c280093c181c2b062a05e0908/B in TestAcidGuarantees,,1731751162662.ffe1934c280093c181c2b062a05e0908. 2024-11-16T09:59:39,116 INFO [RS:0;037c58a19ddb:37267-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/ffe1934c280093c181c2b062a05e0908/A/4f3739c53ae949579bcbf3fa1d7bcc34, hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/ffe1934c280093c181c2b062a05e0908/A/ee2723f2d8864f99892f888682a906e1, hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/ffe1934c280093c181c2b062a05e0908/A/496b53efeb1648a1ae19d0a8097b6c2b] into tmpdir=hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/ffe1934c280093c181c2b062a05e0908/.tmp, totalSize=91.5 K 2024-11-16T09:59:39,116 INFO [RS:0;037c58a19ddb:37267-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/ffe1934c280093c181c2b062a05e0908/B/3b3507fbba454d93a149ba91b104a66e, hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/ffe1934c280093c181c2b062a05e0908/B/ea96bd3b9778485a8591866303bed84e, hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/ffe1934c280093c181c2b062a05e0908/B/7ae7f0d8dc8547829a327b15ba583cf5] into tmpdir=hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/ffe1934c280093c181c2b062a05e0908/.tmp, totalSize=35.9 K 2024-11-16T09:59:39,116 INFO [RS:0;037c58a19ddb:37267-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731751162662.ffe1934c280093c181c2b062a05e0908. 2024-11-16T09:59:39,116 DEBUG [RS:0;037c58a19ddb:37267-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731751162662.ffe1934c280093c181c2b062a05e0908. files: [hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/ffe1934c280093c181c2b062a05e0908/A/4f3739c53ae949579bcbf3fa1d7bcc34, hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/ffe1934c280093c181c2b062a05e0908/A/ee2723f2d8864f99892f888682a906e1, hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/ffe1934c280093c181c2b062a05e0908/A/496b53efeb1648a1ae19d0a8097b6c2b] 2024-11-16T09:59:39,116 DEBUG [RS:0;037c58a19ddb:37267-longCompactions-0 {}] compactions.Compactor(225): Compacting 3b3507fbba454d93a149ba91b104a66e, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=147, earliestPutTs=1731751175027 2024-11-16T09:59:39,116 DEBUG [RS:0;037c58a19ddb:37267-shortCompactions-0 {}] compactions.Compactor(225): Compacting 4f3739c53ae949579bcbf3fa1d7bcc34, keycount=150, bloomtype=ROW, size=30.7 K, encoding=NONE, compression=NONE, seqNum=147, earliestPutTs=1731751175027 2024-11-16T09:59:39,116 DEBUG [RS:0;037c58a19ddb:37267-longCompactions-0 {}] compactions.Compactor(225): Compacting ea96bd3b9778485a8591866303bed84e, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=169, earliestPutTs=1731751175088 2024-11-16T09:59:39,116 DEBUG [RS:0;037c58a19ddb:37267-shortCompactions-0 {}] compactions.Compactor(225): Compacting ee2723f2d8864f99892f888682a906e1, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=169, earliestPutTs=1731751175088 2024-11-16T09:59:39,117 DEBUG [RS:0;037c58a19ddb:37267-shortCompactions-0 {}] compactions.Compactor(225): Compacting 496b53efeb1648a1ae19d0a8097b6c2b, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=187, earliestPutTs=1731751176260 2024-11-16T09:59:39,117 DEBUG [RS:0;037c58a19ddb:37267-longCompactions-0 {}] compactions.Compactor(225): Compacting 7ae7f0d8dc8547829a327b15ba583cf5, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=187, earliestPutTs=1731751176260 2024-11-16T09:59:39,122 INFO [RS:0;037c58a19ddb:37267-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=ffe1934c280093c181c2b062a05e0908] 2024-11-16T09:59:39,123 INFO [RS:0;037c58a19ddb:37267-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): ffe1934c280093c181c2b062a05e0908#B#compaction#339 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-16T09:59:39,123 DEBUG [RS:0;037c58a19ddb:37267-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e2024111619bf6e656e704fb1a4816fd0b76f3d4c_ffe1934c280093c181c2b062a05e0908 store=[table=TestAcidGuarantees family=A region=ffe1934c280093c181c2b062a05e0908] 2024-11-16T09:59:39,124 DEBUG [RS:0;037c58a19ddb:37267-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/ffe1934c280093c181c2b062a05e0908/.tmp/B/32d1270366ac4a4688c1aae590fea7c5 is 50, key is test_row_0/B:col10/1731751176268/Put/seqid=0 2024-11-16T09:59:39,125 DEBUG [RS:0;037c58a19ddb:37267-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e2024111619bf6e656e704fb1a4816fd0b76f3d4c_ffe1934c280093c181c2b062a05e0908, store=[table=TestAcidGuarantees family=A region=ffe1934c280093c181c2b062a05e0908] 2024-11-16T09:59:39,125 DEBUG [RS:0;037c58a19ddb:37267-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111619bf6e656e704fb1a4816fd0b76f3d4c_ffe1934c280093c181c2b062a05e0908 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=ffe1934c280093c181c2b062a05e0908] 2024-11-16T09:59:39,128 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37591 is added to blk_1073742223_1399 (size=12595) 2024-11-16T09:59:39,130 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37591 is added to blk_1073742224_1400 (size=4469) 2024-11-16T09:59:39,131 INFO [RS:0;037c58a19ddb:37267-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): ffe1934c280093c181c2b062a05e0908#A#compaction#338 average throughput is 2.71 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-16T09:59:39,132 DEBUG [RS:0;037c58a19ddb:37267-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/ffe1934c280093c181c2b062a05e0908/.tmp/A/3160dd6fd84e4b82b402bd46540cc5ed is 175, key is test_row_0/A:col10/1731751176268/Put/seqid=0 2024-11-16T09:59:39,135 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37591 is added to blk_1073742225_1401 (size=31549) 2024-11-16T09:59:39,233 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=37267 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=106 2024-11-16T09:59:39,234 DEBUG [RS_FLUSH_OPERATIONS-regionserver/037c58a19ddb:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731751162662.ffe1934c280093c181c2b062a05e0908. 2024-11-16T09:59:39,234 INFO [RS_FLUSH_OPERATIONS-regionserver/037c58a19ddb:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HRegion(2902): Flushing ffe1934c280093c181c2b062a05e0908 3/3 column families, dataSize=127.47 KB heapSize=334.73 KB 2024-11-16T09:59:39,241 DEBUG [RS_FLUSH_OPERATIONS-regionserver/037c58a19ddb:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241116fc79cb79698c4c83bc45db23fbe52c09_ffe1934c280093c181c2b062a05e0908 is 50, key is test_row_0/A:col10/1731751177483/Put/seqid=0 2024-11-16T09:59:39,251 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37591 is added to blk_1073742226_1402 (size=12304) 2024-11-16T09:59:39,533 DEBUG [RS:0;037c58a19ddb:37267-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/ffe1934c280093c181c2b062a05e0908/.tmp/B/32d1270366ac4a4688c1aae590fea7c5 as hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/ffe1934c280093c181c2b062a05e0908/B/32d1270366ac4a4688c1aae590fea7c5 2024-11-16T09:59:39,537 INFO [RS:0;037c58a19ddb:37267-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in ffe1934c280093c181c2b062a05e0908/B of ffe1934c280093c181c2b062a05e0908 into 32d1270366ac4a4688c1aae590fea7c5(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-16T09:59:39,537 DEBUG [RS:0;037c58a19ddb:37267-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for ffe1934c280093c181c2b062a05e0908: 2024-11-16T09:59:39,537 INFO [RS:0;037c58a19ddb:37267-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731751162662.ffe1934c280093c181c2b062a05e0908., storeName=ffe1934c280093c181c2b062a05e0908/B, priority=13, startTime=1731751179114; duration=0sec 2024-11-16T09:59:39,537 DEBUG [RS:0;037c58a19ddb:37267-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-16T09:59:39,537 DEBUG [RS:0;037c58a19ddb:37267-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: ffe1934c280093c181c2b062a05e0908:B 2024-11-16T09:59:39,537 DEBUG [RS:0;037c58a19ddb:37267-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-16T09:59:39,538 DEBUG [RS:0;037c58a19ddb:37267-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36795 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-16T09:59:39,538 DEBUG [RS:0;037c58a19ddb:37267-longCompactions-0 {}] regionserver.HStore(1541): ffe1934c280093c181c2b062a05e0908/C is initiating minor compaction (all files) 2024-11-16T09:59:39,538 INFO [RS:0;037c58a19ddb:37267-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of ffe1934c280093c181c2b062a05e0908/C in TestAcidGuarantees,,1731751162662.ffe1934c280093c181c2b062a05e0908. 2024-11-16T09:59:39,539 INFO [RS:0;037c58a19ddb:37267-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/ffe1934c280093c181c2b062a05e0908/C/ed38a46f975e446c9c74ecd764de8df0, hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/ffe1934c280093c181c2b062a05e0908/C/6d07f7417b00489c94662f29698743f1, hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/ffe1934c280093c181c2b062a05e0908/C/20f06da3d4ca4b91a578717bd9dbaee0] into tmpdir=hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/ffe1934c280093c181c2b062a05e0908/.tmp, totalSize=35.9 K 2024-11-16T09:59:39,539 DEBUG [RS:0;037c58a19ddb:37267-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/ffe1934c280093c181c2b062a05e0908/.tmp/A/3160dd6fd84e4b82b402bd46540cc5ed as hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/ffe1934c280093c181c2b062a05e0908/A/3160dd6fd84e4b82b402bd46540cc5ed 2024-11-16T09:59:39,539 DEBUG [RS:0;037c58a19ddb:37267-longCompactions-0 {}] compactions.Compactor(225): Compacting ed38a46f975e446c9c74ecd764de8df0, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=147, earliestPutTs=1731751175027 2024-11-16T09:59:39,539 DEBUG [RS:0;037c58a19ddb:37267-longCompactions-0 {}] compactions.Compactor(225): Compacting 6d07f7417b00489c94662f29698743f1, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=169, earliestPutTs=1731751175088 2024-11-16T09:59:39,539 DEBUG [RS:0;037c58a19ddb:37267-longCompactions-0 {}] compactions.Compactor(225): Compacting 20f06da3d4ca4b91a578717bd9dbaee0, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=187, earliestPutTs=1731751176260 2024-11-16T09:59:39,542 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37877 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=105 2024-11-16T09:59:39,546 INFO [RS:0;037c58a19ddb:37267-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in ffe1934c280093c181c2b062a05e0908/A of ffe1934c280093c181c2b062a05e0908 into 3160dd6fd84e4b82b402bd46540cc5ed(size=30.8 K), total size for store is 30.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-16T09:59:39,546 DEBUG [RS:0;037c58a19ddb:37267-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for ffe1934c280093c181c2b062a05e0908: 2024-11-16T09:59:39,546 INFO [RS:0;037c58a19ddb:37267-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731751162662.ffe1934c280093c181c2b062a05e0908., storeName=ffe1934c280093c181c2b062a05e0908/A, priority=13, startTime=1731751179114; duration=0sec 2024-11-16T09:59:39,546 DEBUG [RS:0;037c58a19ddb:37267-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-16T09:59:39,546 DEBUG [RS:0;037c58a19ddb:37267-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: ffe1934c280093c181c2b062a05e0908:A 2024-11-16T09:59:39,547 INFO [RS:0;037c58a19ddb:37267-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): ffe1934c280093c181c2b062a05e0908#C#compaction#341 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-16T09:59:39,547 DEBUG [RS:0;037c58a19ddb:37267-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/ffe1934c280093c181c2b062a05e0908/.tmp/C/5ec075b689da412bb397e9b9dbafdfe3 is 50, key is test_row_0/C:col10/1731751176268/Put/seqid=0 2024-11-16T09:59:39,555 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37591 is added to blk_1073742227_1403 (size=12595) 2024-11-16T09:59:39,651 DEBUG [RS_FLUSH_OPERATIONS-regionserver/037c58a19ddb:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T09:59:39,655 INFO [RS_FLUSH_OPERATIONS-regionserver/037c58a19ddb:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241116fc79cb79698c4c83bc45db23fbe52c09_ffe1934c280093c181c2b062a05e0908 to hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241116fc79cb79698c4c83bc45db23fbe52c09_ffe1934c280093c181c2b062a05e0908 2024-11-16T09:59:39,656 DEBUG [RS_FLUSH_OPERATIONS-regionserver/037c58a19ddb:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/ffe1934c280093c181c2b062a05e0908/.tmp/A/8e8ef152dca44dd8a3f29446df0b9790, store: [table=TestAcidGuarantees family=A region=ffe1934c280093c181c2b062a05e0908] 2024-11-16T09:59:39,657 DEBUG [RS_FLUSH_OPERATIONS-regionserver/037c58a19ddb:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/ffe1934c280093c181c2b062a05e0908/.tmp/A/8e8ef152dca44dd8a3f29446df0b9790 is 175, key is test_row_0/A:col10/1731751177483/Put/seqid=0 2024-11-16T09:59:39,657 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] regionserver.HRegion(8855): Flush requested on ffe1934c280093c181c2b062a05e0908 2024-11-16T09:59:39,657 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731751162662.ffe1934c280093c181c2b062a05e0908. as already flushing 2024-11-16T09:59:39,662 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37591 is added to blk_1073742228_1404 (size=31105) 2024-11-16T09:59:39,663 INFO [RS_FLUSH_OPERATIONS-regionserver/037c58a19ddb:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=209, memsize=42.5 K, hasBloomFilter=true, into tmp file hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/ffe1934c280093c181c2b062a05e0908/.tmp/A/8e8ef152dca44dd8a3f29446df0b9790 2024-11-16T09:59:39,671 DEBUG [RS_FLUSH_OPERATIONS-regionserver/037c58a19ddb:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/ffe1934c280093c181c2b062a05e0908/.tmp/B/fa2fc42df2b14c9788de3b052407407a is 50, key is test_row_0/B:col10/1731751177483/Put/seqid=0 2024-11-16T09:59:39,674 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37591 is added to blk_1073742229_1405 (size=12151) 2024-11-16T09:59:39,684 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffe1934c280093c181c2b062a05e0908, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T09:59:39,684 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] ipc.CallRunner(138): callId: 72 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43418 deadline: 1731751239677, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffe1934c280093c181c2b062a05e0908, server=037c58a19ddb,37267,1731751077844 2024-11-16T09:59:39,684 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731751162662.ffe1934c280093c181c2b062a05e0908., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 , the old value is region=TestAcidGuarantees,,1731751162662.ffe1934c280093c181c2b062a05e0908., hostname=037c58a19ddb,37267,1731751077844, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffe1934c280093c181c2b062a05e0908, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:59:39,685 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731751162662.ffe1934c280093c181c2b062a05e0908., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffe1934c280093c181c2b062a05e0908, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:59:39,685 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731751162662.ffe1934c280093c181c2b062a05e0908., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 because the exception is null or not the one we care about 2024-11-16T09:59:39,687 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffe1934c280093c181c2b062a05e0908, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T09:59:39,687 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] ipc.CallRunner(138): callId: 75 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43332 deadline: 1731751239678, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffe1934c280093c181c2b062a05e0908, server=037c58a19ddb,37267,1731751077844 2024-11-16T09:59:39,687 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731751162662.ffe1934c280093c181c2b062a05e0908., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 , the old value is region=TestAcidGuarantees,,1731751162662.ffe1934c280093c181c2b062a05e0908., hostname=037c58a19ddb,37267,1731751077844, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffe1934c280093c181c2b062a05e0908, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:59:39,687 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731751162662.ffe1934c280093c181c2b062a05e0908., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffe1934c280093c181c2b062a05e0908, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:59:39,687 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731751162662.ffe1934c280093c181c2b062a05e0908., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 because the exception is null or not the one we care about 2024-11-16T09:59:39,692 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffe1934c280093c181c2b062a05e0908, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T09:59:39,692 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] ipc.CallRunner(138): callId: 71 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43362 deadline: 1731751239681, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffe1934c280093c181c2b062a05e0908, server=037c58a19ddb,37267,1731751077844 2024-11-16T09:59:39,692 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731751162662.ffe1934c280093c181c2b062a05e0908., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 , the old value is region=TestAcidGuarantees,,1731751162662.ffe1934c280093c181c2b062a05e0908., hostname=037c58a19ddb,37267,1731751077844, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffe1934c280093c181c2b062a05e0908, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:59:39,692 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731751162662.ffe1934c280093c181c2b062a05e0908., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffe1934c280093c181c2b062a05e0908, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:59:39,692 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffe1934c280093c181c2b062a05e0908, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T09:59:39,692 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731751162662.ffe1934c280093c181c2b062a05e0908., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 because the exception is null or not the one we care about 2024-11-16T09:59:39,692 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] ipc.CallRunner(138): callId: 76 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43402 deadline: 1731751239682, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffe1934c280093c181c2b062a05e0908, server=037c58a19ddb,37267,1731751077844 2024-11-16T09:59:39,692 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731751162662.ffe1934c280093c181c2b062a05e0908., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 , the old value is region=TestAcidGuarantees,,1731751162662.ffe1934c280093c181c2b062a05e0908., hostname=037c58a19ddb,37267,1731751077844, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffe1934c280093c181c2b062a05e0908, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:59:39,692 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731751162662.ffe1934c280093c181c2b062a05e0908., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffe1934c280093c181c2b062a05e0908, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:59:39,692 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffe1934c280093c181c2b062a05e0908, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T09:59:39,692 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731751162662.ffe1934c280093c181c2b062a05e0908., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 because the exception is null or not the one we care about 2024-11-16T09:59:39,693 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] ipc.CallRunner(138): callId: 75 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43416 deadline: 1731751239683, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffe1934c280093c181c2b062a05e0908, server=037c58a19ddb,37267,1731751077844 2024-11-16T09:59:39,693 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731751162662.ffe1934c280093c181c2b062a05e0908., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 , the old value is region=TestAcidGuarantees,,1731751162662.ffe1934c280093c181c2b062a05e0908., hostname=037c58a19ddb,37267,1731751077844, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffe1934c280093c181c2b062a05e0908, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:59:39,693 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731751162662.ffe1934c280093c181c2b062a05e0908., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffe1934c280093c181c2b062a05e0908, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:59:39,693 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731751162662.ffe1934c280093c181c2b062a05e0908., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 because the exception is null or not the one we care about 2024-11-16T09:59:39,799 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffe1934c280093c181c2b062a05e0908, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T09:59:39,799 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] ipc.CallRunner(138): callId: 73 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43418 deadline: 1731751239793, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffe1934c280093c181c2b062a05e0908, server=037c58a19ddb,37267,1731751077844 2024-11-16T09:59:39,800 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731751162662.ffe1934c280093c181c2b062a05e0908., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 , the old value is region=TestAcidGuarantees,,1731751162662.ffe1934c280093c181c2b062a05e0908., hostname=037c58a19ddb,37267,1731751077844, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffe1934c280093c181c2b062a05e0908, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:59:39,800 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffe1934c280093c181c2b062a05e0908, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T09:59:39,800 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731751162662.ffe1934c280093c181c2b062a05e0908., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffe1934c280093c181c2b062a05e0908, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:59:39,800 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731751162662.ffe1934c280093c181c2b062a05e0908., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 because the exception is null or not the one we care about 2024-11-16T09:59:39,800 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] ipc.CallRunner(138): callId: 76 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43332 deadline: 1731751239793, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffe1934c280093c181c2b062a05e0908, server=037c58a19ddb,37267,1731751077844 2024-11-16T09:59:39,800 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731751162662.ffe1934c280093c181c2b062a05e0908., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 , the old value is region=TestAcidGuarantees,,1731751162662.ffe1934c280093c181c2b062a05e0908., hostname=037c58a19ddb,37267,1731751077844, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffe1934c280093c181c2b062a05e0908, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:59:39,800 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731751162662.ffe1934c280093c181c2b062a05e0908., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffe1934c280093c181c2b062a05e0908, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:59:39,800 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731751162662.ffe1934c280093c181c2b062a05e0908., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 because the exception is null or not the one we care about 2024-11-16T09:59:39,810 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffe1934c280093c181c2b062a05e0908, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T09:59:39,810 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] ipc.CallRunner(138): callId: 72 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43362 deadline: 1731751239802, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffe1934c280093c181c2b062a05e0908, server=037c58a19ddb,37267,1731751077844 2024-11-16T09:59:39,811 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffe1934c280093c181c2b062a05e0908, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T09:59:39,811 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731751162662.ffe1934c280093c181c2b062a05e0908., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 , the old value is region=TestAcidGuarantees,,1731751162662.ffe1934c280093c181c2b062a05e0908., hostname=037c58a19ddb,37267,1731751077844, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffe1934c280093c181c2b062a05e0908, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:59:39,811 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731751162662.ffe1934c280093c181c2b062a05e0908., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffe1934c280093c181c2b062a05e0908, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:59:39,811 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731751162662.ffe1934c280093c181c2b062a05e0908., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 because the exception is null or not the one we care about 2024-11-16T09:59:39,811 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] ipc.CallRunner(138): callId: 77 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43402 deadline: 1731751239803, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffe1934c280093c181c2b062a05e0908, server=037c58a19ddb,37267,1731751077844 2024-11-16T09:59:39,811 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffe1934c280093c181c2b062a05e0908, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T09:59:39,811 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731751162662.ffe1934c280093c181c2b062a05e0908., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 , the old value is region=TestAcidGuarantees,,1731751162662.ffe1934c280093c181c2b062a05e0908., hostname=037c58a19ddb,37267,1731751077844, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffe1934c280093c181c2b062a05e0908, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:59:39,811 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731751162662.ffe1934c280093c181c2b062a05e0908., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffe1934c280093c181c2b062a05e0908, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:59:39,811 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] ipc.CallRunner(138): callId: 76 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43416 deadline: 1731751239803, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffe1934c280093c181c2b062a05e0908, server=037c58a19ddb,37267,1731751077844 2024-11-16T09:59:39,811 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731751162662.ffe1934c280093c181c2b062a05e0908., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 because the exception is null or not the one we care about 2024-11-16T09:59:39,811 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731751162662.ffe1934c280093c181c2b062a05e0908., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 , the old value is region=TestAcidGuarantees,,1731751162662.ffe1934c280093c181c2b062a05e0908., hostname=037c58a19ddb,37267,1731751077844, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffe1934c280093c181c2b062a05e0908, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:59:39,811 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731751162662.ffe1934c280093c181c2b062a05e0908., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffe1934c280093c181c2b062a05e0908, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:59:39,811 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731751162662.ffe1934c280093c181c2b062a05e0908., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 because the exception is null or not the one we care about 2024-11-16T09:59:39,961 DEBUG [RS:0;037c58a19ddb:37267-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/ffe1934c280093c181c2b062a05e0908/.tmp/C/5ec075b689da412bb397e9b9dbafdfe3 as hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/ffe1934c280093c181c2b062a05e0908/C/5ec075b689da412bb397e9b9dbafdfe3 2024-11-16T09:59:39,965 INFO [RS:0;037c58a19ddb:37267-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in ffe1934c280093c181c2b062a05e0908/C of ffe1934c280093c181c2b062a05e0908 into 5ec075b689da412bb397e9b9dbafdfe3(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-16T09:59:39,965 DEBUG [RS:0;037c58a19ddb:37267-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for ffe1934c280093c181c2b062a05e0908: 2024-11-16T09:59:39,965 INFO [RS:0;037c58a19ddb:37267-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731751162662.ffe1934c280093c181c2b062a05e0908., storeName=ffe1934c280093c181c2b062a05e0908/C, priority=13, startTime=1731751179114; duration=0sec 2024-11-16T09:59:39,965 DEBUG [RS:0;037c58a19ddb:37267-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-16T09:59:39,966 DEBUG [RS:0;037c58a19ddb:37267-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: ffe1934c280093c181c2b062a05e0908:C 2024-11-16T09:59:40,005 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffe1934c280093c181c2b062a05e0908, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T09:59:40,005 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] ipc.CallRunner(138): callId: 74 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43418 deadline: 1731751240002, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffe1934c280093c181c2b062a05e0908, server=037c58a19ddb,37267,1731751077844 2024-11-16T09:59:40,005 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731751162662.ffe1934c280093c181c2b062a05e0908., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 , the old value is region=TestAcidGuarantees,,1731751162662.ffe1934c280093c181c2b062a05e0908., hostname=037c58a19ddb,37267,1731751077844, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffe1934c280093c181c2b062a05e0908, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:59:40,005 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731751162662.ffe1934c280093c181c2b062a05e0908., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffe1934c280093c181c2b062a05e0908, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:59:40,005 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731751162662.ffe1934c280093c181c2b062a05e0908., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 because the exception is null or not the one we care about 2024-11-16T09:59:40,020 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffe1934c280093c181c2b062a05e0908, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T09:59:40,020 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] ipc.CallRunner(138): callId: 77 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43332 deadline: 1731751240013, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffe1934c280093c181c2b062a05e0908, server=037c58a19ddb,37267,1731751077844 2024-11-16T09:59:40,020 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffe1934c280093c181c2b062a05e0908, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T09:59:40,021 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] ipc.CallRunner(138): callId: 77 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43416 deadline: 1731751240013, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffe1934c280093c181c2b062a05e0908, server=037c58a19ddb,37267,1731751077844 2024-11-16T09:59:40,021 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731751162662.ffe1934c280093c181c2b062a05e0908., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 , the old value is region=TestAcidGuarantees,,1731751162662.ffe1934c280093c181c2b062a05e0908., hostname=037c58a19ddb,37267,1731751077844, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffe1934c280093c181c2b062a05e0908, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:59:40,021 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731751162662.ffe1934c280093c181c2b062a05e0908., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffe1934c280093c181c2b062a05e0908, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:59:40,021 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731751162662.ffe1934c280093c181c2b062a05e0908., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 because the exception is null or not the one we care about 2024-11-16T09:59:40,021 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffe1934c280093c181c2b062a05e0908, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T09:59:40,021 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731751162662.ffe1934c280093c181c2b062a05e0908., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 , the old value is region=TestAcidGuarantees,,1731751162662.ffe1934c280093c181c2b062a05e0908., hostname=037c58a19ddb,37267,1731751077844, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffe1934c280093c181c2b062a05e0908, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:59:40,021 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731751162662.ffe1934c280093c181c2b062a05e0908., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffe1934c280093c181c2b062a05e0908, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:59:40,021 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] ipc.CallRunner(138): callId: 78 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43402 deadline: 1731751240013, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffe1934c280093c181c2b062a05e0908, server=037c58a19ddb,37267,1731751077844 2024-11-16T09:59:40,021 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731751162662.ffe1934c280093c181c2b062a05e0908., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 because the exception is null or not the one we care about 2024-11-16T09:59:40,021 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731751162662.ffe1934c280093c181c2b062a05e0908., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 , the old value is region=TestAcidGuarantees,,1731751162662.ffe1934c280093c181c2b062a05e0908., hostname=037c58a19ddb,37267,1731751077844, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffe1934c280093c181c2b062a05e0908, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:59:40,021 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731751162662.ffe1934c280093c181c2b062a05e0908., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffe1934c280093c181c2b062a05e0908, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:59:40,021 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731751162662.ffe1934c280093c181c2b062a05e0908., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 because the exception is null or not the one we care about 2024-11-16T09:59:40,031 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffe1934c280093c181c2b062a05e0908, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T09:59:40,031 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] ipc.CallRunner(138): callId: 73 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43362 deadline: 1731751240023, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffe1934c280093c181c2b062a05e0908, server=037c58a19ddb,37267,1731751077844 2024-11-16T09:59:40,031 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731751162662.ffe1934c280093c181c2b062a05e0908., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 , the old value is region=TestAcidGuarantees,,1731751162662.ffe1934c280093c181c2b062a05e0908., hostname=037c58a19ddb,37267,1731751077844, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffe1934c280093c181c2b062a05e0908, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:59:40,031 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731751162662.ffe1934c280093c181c2b062a05e0908., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffe1934c280093c181c2b062a05e0908, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:59:40,031 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731751162662.ffe1934c280093c181c2b062a05e0908., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 because the exception is null or not the one we care about 2024-11-16T09:59:40,075 INFO [RS_FLUSH_OPERATIONS-regionserver/037c58a19ddb:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=209 (bloomFilter=true), to=hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/ffe1934c280093c181c2b062a05e0908/.tmp/B/fa2fc42df2b14c9788de3b052407407a 2024-11-16T09:59:40,083 DEBUG [RS_FLUSH_OPERATIONS-regionserver/037c58a19ddb:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/ffe1934c280093c181c2b062a05e0908/.tmp/C/a0afb7c2bda64ca680e563efa5618949 is 50, key is test_row_0/C:col10/1731751177483/Put/seqid=0 2024-11-16T09:59:40,087 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37591 is added to blk_1073742230_1406 (size=12151) 2024-11-16T09:59:40,320 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffe1934c280093c181c2b062a05e0908, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T09:59:40,320 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] ipc.CallRunner(138): callId: 75 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43418 deadline: 1731751240313, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffe1934c280093c181c2b062a05e0908, server=037c58a19ddb,37267,1731751077844 2024-11-16T09:59:40,320 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731751162662.ffe1934c280093c181c2b062a05e0908., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 , the old value is region=TestAcidGuarantees,,1731751162662.ffe1934c280093c181c2b062a05e0908., hostname=037c58a19ddb,37267,1731751077844, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffe1934c280093c181c2b062a05e0908, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:59:40,320 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731751162662.ffe1934c280093c181c2b062a05e0908., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffe1934c280093c181c2b062a05e0908, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:59:40,320 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731751162662.ffe1934c280093c181c2b062a05e0908., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 because the exception is null or not the one we care about 2024-11-16T09:59:40,330 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffe1934c280093c181c2b062a05e0908, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T09:59:40,330 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] ipc.CallRunner(138): callId: 78 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43416 deadline: 1731751240322, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffe1934c280093c181c2b062a05e0908, server=037c58a19ddb,37267,1731751077844 2024-11-16T09:59:40,330 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731751162662.ffe1934c280093c181c2b062a05e0908., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 , the old value is region=TestAcidGuarantees,,1731751162662.ffe1934c280093c181c2b062a05e0908., hostname=037c58a19ddb,37267,1731751077844, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffe1934c280093c181c2b062a05e0908, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:59:40,330 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731751162662.ffe1934c280093c181c2b062a05e0908., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffe1934c280093c181c2b062a05e0908, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:59:40,330 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731751162662.ffe1934c280093c181c2b062a05e0908., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 because the exception is null or not the one we care about 2024-11-16T09:59:40,340 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffe1934c280093c181c2b062a05e0908, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T09:59:40,340 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] ipc.CallRunner(138): callId: 78 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43332 deadline: 1731751240333, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffe1934c280093c181c2b062a05e0908, server=037c58a19ddb,37267,1731751077844 2024-11-16T09:59:40,340 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffe1934c280093c181c2b062a05e0908, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T09:59:40,340 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] ipc.CallRunner(138): callId: 79 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43402 deadline: 1731751240333, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffe1934c280093c181c2b062a05e0908, server=037c58a19ddb,37267,1731751077844 2024-11-16T09:59:40,340 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731751162662.ffe1934c280093c181c2b062a05e0908., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 , the old value is region=TestAcidGuarantees,,1731751162662.ffe1934c280093c181c2b062a05e0908., hostname=037c58a19ddb,37267,1731751077844, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffe1934c280093c181c2b062a05e0908, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:59:40,340 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731751162662.ffe1934c280093c181c2b062a05e0908., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffe1934c280093c181c2b062a05e0908, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:59:40,340 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731751162662.ffe1934c280093c181c2b062a05e0908., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 because the exception is null or not the one we care about 2024-11-16T09:59:40,340 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731751162662.ffe1934c280093c181c2b062a05e0908., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 , the old value is region=TestAcidGuarantees,,1731751162662.ffe1934c280093c181c2b062a05e0908., hostname=037c58a19ddb,37267,1731751077844, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffe1934c280093c181c2b062a05e0908, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:59:40,341 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731751162662.ffe1934c280093c181c2b062a05e0908., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffe1934c280093c181c2b062a05e0908, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:59:40,341 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731751162662.ffe1934c280093c181c2b062a05e0908., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 because the exception is null or not the one we care about 2024-11-16T09:59:40,350 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffe1934c280093c181c2b062a05e0908, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T09:59:40,350 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] ipc.CallRunner(138): callId: 74 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43362 deadline: 1731751240343, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffe1934c280093c181c2b062a05e0908, server=037c58a19ddb,37267,1731751077844 2024-11-16T09:59:40,350 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731751162662.ffe1934c280093c181c2b062a05e0908., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 , the old value is region=TestAcidGuarantees,,1731751162662.ffe1934c280093c181c2b062a05e0908., hostname=037c58a19ddb,37267,1731751077844, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffe1934c280093c181c2b062a05e0908, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:59:40,350 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731751162662.ffe1934c280093c181c2b062a05e0908., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffe1934c280093c181c2b062a05e0908, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:59:40,350 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731751162662.ffe1934c280093c181c2b062a05e0908., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 because the exception is null or not the one we care about 2024-11-16T09:59:40,487 INFO [RS_FLUSH_OPERATIONS-regionserver/037c58a19ddb:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=209 (bloomFilter=true), to=hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/ffe1934c280093c181c2b062a05e0908/.tmp/C/a0afb7c2bda64ca680e563efa5618949 2024-11-16T09:59:40,493 DEBUG [RS_FLUSH_OPERATIONS-regionserver/037c58a19ddb:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/ffe1934c280093c181c2b062a05e0908/.tmp/A/8e8ef152dca44dd8a3f29446df0b9790 as hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/ffe1934c280093c181c2b062a05e0908/A/8e8ef152dca44dd8a3f29446df0b9790 2024-11-16T09:59:40,496 INFO [RS_FLUSH_OPERATIONS-regionserver/037c58a19ddb:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/ffe1934c280093c181c2b062a05e0908/A/8e8ef152dca44dd8a3f29446df0b9790, entries=150, sequenceid=209, filesize=30.4 K 2024-11-16T09:59:40,497 DEBUG [RS_FLUSH_OPERATIONS-regionserver/037c58a19ddb:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/ffe1934c280093c181c2b062a05e0908/.tmp/B/fa2fc42df2b14c9788de3b052407407a as hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/ffe1934c280093c181c2b062a05e0908/B/fa2fc42df2b14c9788de3b052407407a 2024-11-16T09:59:40,501 INFO [RS_FLUSH_OPERATIONS-regionserver/037c58a19ddb:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/ffe1934c280093c181c2b062a05e0908/B/fa2fc42df2b14c9788de3b052407407a, entries=150, sequenceid=209, filesize=11.9 K 2024-11-16T09:59:40,502 DEBUG [RS_FLUSH_OPERATIONS-regionserver/037c58a19ddb:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/ffe1934c280093c181c2b062a05e0908/.tmp/C/a0afb7c2bda64ca680e563efa5618949 as hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/ffe1934c280093c181c2b062a05e0908/C/a0afb7c2bda64ca680e563efa5618949 2024-11-16T09:59:40,505 INFO [RS_FLUSH_OPERATIONS-regionserver/037c58a19ddb:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/ffe1934c280093c181c2b062a05e0908/C/a0afb7c2bda64ca680e563efa5618949, entries=150, sequenceid=209, filesize=11.9 K 2024-11-16T09:59:40,505 INFO [RS_FLUSH_OPERATIONS-regionserver/037c58a19ddb:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HRegion(3140): Finished flush of dataSize ~127.47 KB/130530, heapSize ~334.69 KB/342720, currentSize=73.80 KB/75570 for ffe1934c280093c181c2b062a05e0908 in 1271ms, sequenceid=209, compaction requested=false 2024-11-16T09:59:40,506 DEBUG [RS_FLUSH_OPERATIONS-regionserver/037c58a19ddb:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HRegion(2603): Flush status journal for ffe1934c280093c181c2b062a05e0908: 2024-11-16T09:59:40,506 DEBUG [RS_FLUSH_OPERATIONS-regionserver/037c58a19ddb:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731751162662.ffe1934c280093c181c2b062a05e0908. 2024-11-16T09:59:40,506 DEBUG [RS_FLUSH_OPERATIONS-regionserver/037c58a19ddb:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=106 2024-11-16T09:59:40,506 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37877 {}] master.HMaster(4169): Remote procedure done, pid=106 2024-11-16T09:59:40,517 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=106, resume processing ppid=105 2024-11-16T09:59:40,517 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=106, ppid=105, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 3.1110 sec 2024-11-16T09:59:40,519 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=105, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=105, table=TestAcidGuarantees in 3.1240 sec 2024-11-16T09:59:40,837 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] regionserver.HRegion(8855): Flush requested on ffe1934c280093c181c2b062a05e0908 2024-11-16T09:59:40,837 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-16T09:59:40,837 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing ffe1934c280093c181c2b062a05e0908 3/3 column families, dataSize=80.51 KB heapSize=211.69 KB 2024-11-16T09:59:40,844 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411162584b4e9ec0542f2ae8795cee9ff949a_ffe1934c280093c181c2b062a05e0908 is 50, key is test_row_0/A:col10/1731751179682/Put/seqid=0 2024-11-16T09:59:40,848 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37591 is added to blk_1073742231_1407 (size=14794) 2024-11-16T09:59:40,887 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffe1934c280093c181c2b062a05e0908, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T09:59:40,887 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] ipc.CallRunner(138): callId: 81 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43332 deadline: 1731751240877, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffe1934c280093c181c2b062a05e0908, server=037c58a19ddb,37267,1731751077844 2024-11-16T09:59:40,887 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffe1934c280093c181c2b062a05e0908, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T09:59:40,887 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] ipc.CallRunner(138): callId: 82 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43402 deadline: 1731751240877, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffe1934c280093c181c2b062a05e0908, server=037c58a19ddb,37267,1731751077844 2024-11-16T09:59:40,887 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731751162662.ffe1934c280093c181c2b062a05e0908., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 , the old value is region=TestAcidGuarantees,,1731751162662.ffe1934c280093c181c2b062a05e0908., hostname=037c58a19ddb,37267,1731751077844, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffe1934c280093c181c2b062a05e0908, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:59:40,887 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731751162662.ffe1934c280093c181c2b062a05e0908., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffe1934c280093c181c2b062a05e0908, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:59:40,887 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffe1934c280093c181c2b062a05e0908, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T09:59:40,887 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731751162662.ffe1934c280093c181c2b062a05e0908., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 , the old value is region=TestAcidGuarantees,,1731751162662.ffe1934c280093c181c2b062a05e0908., hostname=037c58a19ddb,37267,1731751077844, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffe1934c280093c181c2b062a05e0908, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:59:40,887 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731751162662.ffe1934c280093c181c2b062a05e0908., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 because the exception is null or not the one we care about 2024-11-16T09:59:40,887 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731751162662.ffe1934c280093c181c2b062a05e0908., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffe1934c280093c181c2b062a05e0908, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:59:40,888 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731751162662.ffe1934c280093c181c2b062a05e0908., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 because the exception is null or not the one we care about 2024-11-16T09:59:40,888 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] ipc.CallRunner(138): callId: 83 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43418 deadline: 1731751240878, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffe1934c280093c181c2b062a05e0908, server=037c58a19ddb,37267,1731751077844 2024-11-16T09:59:40,888 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffe1934c280093c181c2b062a05e0908, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T09:59:40,888 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] ipc.CallRunner(138): callId: 85 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43416 deadline: 1731751240880, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffe1934c280093c181c2b062a05e0908, server=037c58a19ddb,37267,1731751077844 2024-11-16T09:59:40,888 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731751162662.ffe1934c280093c181c2b062a05e0908., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 , the old value is region=TestAcidGuarantees,,1731751162662.ffe1934c280093c181c2b062a05e0908., hostname=037c58a19ddb,37267,1731751077844, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffe1934c280093c181c2b062a05e0908, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:59:40,888 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731751162662.ffe1934c280093c181c2b062a05e0908., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffe1934c280093c181c2b062a05e0908, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:59:40,888 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731751162662.ffe1934c280093c181c2b062a05e0908., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 because the exception is null or not the one we care about 2024-11-16T09:59:40,888 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731751162662.ffe1934c280093c181c2b062a05e0908., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 , the old value is region=TestAcidGuarantees,,1731751162662.ffe1934c280093c181c2b062a05e0908., hostname=037c58a19ddb,37267,1731751077844, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffe1934c280093c181c2b062a05e0908, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:59:40,888 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731751162662.ffe1934c280093c181c2b062a05e0908., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffe1934c280093c181c2b062a05e0908, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:59:40,888 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731751162662.ffe1934c280093c181c2b062a05e0908., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 because the exception is null or not the one we care about 2024-11-16T09:59:40,892 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffe1934c280093c181c2b062a05e0908, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T09:59:40,892 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] ipc.CallRunner(138): callId: 77 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43362 deadline: 1731751240882, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffe1934c280093c181c2b062a05e0908, server=037c58a19ddb,37267,1731751077844 2024-11-16T09:59:40,893 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731751162662.ffe1934c280093c181c2b062a05e0908., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 , the old value is region=TestAcidGuarantees,,1731751162662.ffe1934c280093c181c2b062a05e0908., hostname=037c58a19ddb,37267,1731751077844, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffe1934c280093c181c2b062a05e0908, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:59:40,893 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731751162662.ffe1934c280093c181c2b062a05e0908., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffe1934c280093c181c2b062a05e0908, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:59:40,893 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731751162662.ffe1934c280093c181c2b062a05e0908., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 because the exception is null or not the one we care about 2024-11-16T09:59:40,998 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffe1934c280093c181c2b062a05e0908, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T09:59:40,998 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] ipc.CallRunner(138): callId: 82 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43332 deadline: 1731751240992, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffe1934c280093c181c2b062a05e0908, server=037c58a19ddb,37267,1731751077844 2024-11-16T09:59:40,998 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffe1934c280093c181c2b062a05e0908, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T09:59:40,998 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] ipc.CallRunner(138): callId: 83 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43402 deadline: 1731751240992, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffe1934c280093c181c2b062a05e0908, server=037c58a19ddb,37267,1731751077844 2024-11-16T09:59:40,998 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731751162662.ffe1934c280093c181c2b062a05e0908., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 , the old value is region=TestAcidGuarantees,,1731751162662.ffe1934c280093c181c2b062a05e0908., hostname=037c58a19ddb,37267,1731751077844, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffe1934c280093c181c2b062a05e0908, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:59:40,998 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731751162662.ffe1934c280093c181c2b062a05e0908., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffe1934c280093c181c2b062a05e0908, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:59:40,998 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731751162662.ffe1934c280093c181c2b062a05e0908., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 because the exception is null or not the one we care about 2024-11-16T09:59:40,998 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffe1934c280093c181c2b062a05e0908, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T09:59:40,998 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731751162662.ffe1934c280093c181c2b062a05e0908., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 , the old value is region=TestAcidGuarantees,,1731751162662.ffe1934c280093c181c2b062a05e0908., hostname=037c58a19ddb,37267,1731751077844, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffe1934c280093c181c2b062a05e0908, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:59:40,998 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731751162662.ffe1934c280093c181c2b062a05e0908., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffe1934c280093c181c2b062a05e0908, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:59:40,998 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731751162662.ffe1934c280093c181c2b062a05e0908., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 because the exception is null or not the one we care about 2024-11-16T09:59:40,998 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] ipc.CallRunner(138): callId: 84 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43418 deadline: 1731751240993, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffe1934c280093c181c2b062a05e0908, server=037c58a19ddb,37267,1731751077844 2024-11-16T09:59:40,999 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffe1934c280093c181c2b062a05e0908, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T09:59:40,999 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731751162662.ffe1934c280093c181c2b062a05e0908., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 , the old value is region=TestAcidGuarantees,,1731751162662.ffe1934c280093c181c2b062a05e0908., hostname=037c58a19ddb,37267,1731751077844, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffe1934c280093c181c2b062a05e0908, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:59:40,999 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] ipc.CallRunner(138): callId: 86 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43416 deadline: 1731751240993, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffe1934c280093c181c2b062a05e0908, server=037c58a19ddb,37267,1731751077844 2024-11-16T09:59:40,999 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731751162662.ffe1934c280093c181c2b062a05e0908., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffe1934c280093c181c2b062a05e0908, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:59:40,999 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731751162662.ffe1934c280093c181c2b062a05e0908., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 because the exception is null or not the one we care about 2024-11-16T09:59:40,999 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731751162662.ffe1934c280093c181c2b062a05e0908., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 , the old value is region=TestAcidGuarantees,,1731751162662.ffe1934c280093c181c2b062a05e0908., hostname=037c58a19ddb,37267,1731751077844, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffe1934c280093c181c2b062a05e0908, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:59:40,999 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731751162662.ffe1934c280093c181c2b062a05e0908., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffe1934c280093c181c2b062a05e0908, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:59:40,999 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731751162662.ffe1934c280093c181c2b062a05e0908., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 because the exception is null or not the one we care about 2024-11-16T09:59:41,009 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffe1934c280093c181c2b062a05e0908, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T09:59:41,009 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] ipc.CallRunner(138): callId: 78 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43362 deadline: 1731751241003, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffe1934c280093c181c2b062a05e0908, server=037c58a19ddb,37267,1731751077844 2024-11-16T09:59:41,009 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731751162662.ffe1934c280093c181c2b062a05e0908., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 , the old value is region=TestAcidGuarantees,,1731751162662.ffe1934c280093c181c2b062a05e0908., hostname=037c58a19ddb,37267,1731751077844, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffe1934c280093c181c2b062a05e0908, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:59:41,009 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731751162662.ffe1934c280093c181c2b062a05e0908., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffe1934c280093c181c2b062a05e0908, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:59:41,009 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731751162662.ffe1934c280093c181c2b062a05e0908., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 because the exception is null or not the one we care about 2024-11-16T09:59:41,209 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffe1934c280093c181c2b062a05e0908, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T09:59:41,209 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] ipc.CallRunner(138): callId: 83 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43332 deadline: 1731751241202, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffe1934c280093c181c2b062a05e0908, server=037c58a19ddb,37267,1731751077844 2024-11-16T09:59:41,209 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffe1934c280093c181c2b062a05e0908, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T09:59:41,209 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] ipc.CallRunner(138): callId: 85 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43418 deadline: 1731751241203, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffe1934c280093c181c2b062a05e0908, server=037c58a19ddb,37267,1731751077844 2024-11-16T09:59:41,209 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731751162662.ffe1934c280093c181c2b062a05e0908., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 , the old value is region=TestAcidGuarantees,,1731751162662.ffe1934c280093c181c2b062a05e0908., hostname=037c58a19ddb,37267,1731751077844, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffe1934c280093c181c2b062a05e0908, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:59:41,209 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731751162662.ffe1934c280093c181c2b062a05e0908., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffe1934c280093c181c2b062a05e0908, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:59:41,209 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731751162662.ffe1934c280093c181c2b062a05e0908., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 because the exception is null or not the one we care about 2024-11-16T09:59:41,209 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffe1934c280093c181c2b062a05e0908, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T09:59:41,210 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731751162662.ffe1934c280093c181c2b062a05e0908., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 , the old value is region=TestAcidGuarantees,,1731751162662.ffe1934c280093c181c2b062a05e0908., hostname=037c58a19ddb,37267,1731751077844, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffe1934c280093c181c2b062a05e0908, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:59:41,210 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] ipc.CallRunner(138): callId: 84 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43402 deadline: 1731751241203, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffe1934c280093c181c2b062a05e0908, server=037c58a19ddb,37267,1731751077844 2024-11-16T09:59:41,210 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731751162662.ffe1934c280093c181c2b062a05e0908., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffe1934c280093c181c2b062a05e0908, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:59:41,210 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731751162662.ffe1934c280093c181c2b062a05e0908., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 because the exception is null or not the one we care about 2024-11-16T09:59:41,210 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731751162662.ffe1934c280093c181c2b062a05e0908., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 , the old value is region=TestAcidGuarantees,,1731751162662.ffe1934c280093c181c2b062a05e0908., hostname=037c58a19ddb,37267,1731751077844, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffe1934c280093c181c2b062a05e0908, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:59:41,210 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffe1934c280093c181c2b062a05e0908, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T09:59:41,210 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731751162662.ffe1934c280093c181c2b062a05e0908., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffe1934c280093c181c2b062a05e0908, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:59:41,210 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731751162662.ffe1934c280093c181c2b062a05e0908., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 because the exception is null or not the one we care about 2024-11-16T09:59:41,210 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] ipc.CallRunner(138): callId: 87 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43416 deadline: 1731751241203, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffe1934c280093c181c2b062a05e0908, server=037c58a19ddb,37267,1731751077844 2024-11-16T09:59:41,210 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731751162662.ffe1934c280093c181c2b062a05e0908., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 , the old value is region=TestAcidGuarantees,,1731751162662.ffe1934c280093c181c2b062a05e0908., hostname=037c58a19ddb,37267,1731751077844, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffe1934c280093c181c2b062a05e0908, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:59:41,210 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731751162662.ffe1934c280093c181c2b062a05e0908., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffe1934c280093c181c2b062a05e0908, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:59:41,210 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731751162662.ffe1934c280093c181c2b062a05e0908., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 because the exception is null or not the one we care about 2024-11-16T09:59:41,220 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffe1934c280093c181c2b062a05e0908, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T09:59:41,220 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] ipc.CallRunner(138): callId: 79 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43362 deadline: 1731751241213, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffe1934c280093c181c2b062a05e0908, server=037c58a19ddb,37267,1731751077844 2024-11-16T09:59:41,221 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731751162662.ffe1934c280093c181c2b062a05e0908., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 , the old value is region=TestAcidGuarantees,,1731751162662.ffe1934c280093c181c2b062a05e0908., hostname=037c58a19ddb,37267,1731751077844, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffe1934c280093c181c2b062a05e0908, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:59:41,221 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731751162662.ffe1934c280093c181c2b062a05e0908., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffe1934c280093c181c2b062a05e0908, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:59:41,221 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731751162662.ffe1934c280093c181c2b062a05e0908., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 because the exception is null or not the one we care about 2024-11-16T09:59:41,249 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T09:59:41,252 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411162584b4e9ec0542f2ae8795cee9ff949a_ffe1934c280093c181c2b062a05e0908 to hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411162584b4e9ec0542f2ae8795cee9ff949a_ffe1934c280093c181c2b062a05e0908 2024-11-16T09:59:41,253 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/ffe1934c280093c181c2b062a05e0908/.tmp/A/9e8350a74f8a4165b66b073a641f4577, store: [table=TestAcidGuarantees family=A region=ffe1934c280093c181c2b062a05e0908] 2024-11-16T09:59:41,254 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/ffe1934c280093c181c2b062a05e0908/.tmp/A/9e8350a74f8a4165b66b073a641f4577 is 175, key is test_row_0/A:col10/1731751179682/Put/seqid=0 2024-11-16T09:59:41,258 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37591 is added to blk_1073742232_1408 (size=39749) 2024-11-16T09:59:41,517 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffe1934c280093c181c2b062a05e0908, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T09:59:41,517 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] ipc.CallRunner(138): callId: 86 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43418 deadline: 1731751241513, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffe1934c280093c181c2b062a05e0908, server=037c58a19ddb,37267,1731751077844 2024-11-16T09:59:41,518 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731751162662.ffe1934c280093c181c2b062a05e0908., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 , the old value is region=TestAcidGuarantees,,1731751162662.ffe1934c280093c181c2b062a05e0908., hostname=037c58a19ddb,37267,1731751077844, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffe1934c280093c181c2b062a05e0908, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:59:41,518 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731751162662.ffe1934c280093c181c2b062a05e0908., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffe1934c280093c181c2b062a05e0908, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:59:41,518 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731751162662.ffe1934c280093c181c2b062a05e0908., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 because the exception is null or not the one we care about 2024-11-16T09:59:41,528 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffe1934c280093c181c2b062a05e0908, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T09:59:41,528 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] ipc.CallRunner(138): callId: 84 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43332 deadline: 1731751241523, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffe1934c280093c181c2b062a05e0908, server=037c58a19ddb,37267,1731751077844 2024-11-16T09:59:41,528 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731751162662.ffe1934c280093c181c2b062a05e0908., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 , the old value is region=TestAcidGuarantees,,1731751162662.ffe1934c280093c181c2b062a05e0908., hostname=037c58a19ddb,37267,1731751077844, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffe1934c280093c181c2b062a05e0908, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:59:41,528 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffe1934c280093c181c2b062a05e0908, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T09:59:41,528 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731751162662.ffe1934c280093c181c2b062a05e0908., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffe1934c280093c181c2b062a05e0908, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:59:41,528 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731751162662.ffe1934c280093c181c2b062a05e0908., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 because the exception is null or not the one we care about 2024-11-16T09:59:41,528 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] ipc.CallRunner(138): callId: 85 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43402 deadline: 1731751241523, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffe1934c280093c181c2b062a05e0908, server=037c58a19ddb,37267,1731751077844 2024-11-16T09:59:41,529 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffe1934c280093c181c2b062a05e0908, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T09:59:41,529 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731751162662.ffe1934c280093c181c2b062a05e0908., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 , the old value is region=TestAcidGuarantees,,1731751162662.ffe1934c280093c181c2b062a05e0908., hostname=037c58a19ddb,37267,1731751077844, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffe1934c280093c181c2b062a05e0908, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:59:41,529 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] ipc.CallRunner(138): callId: 88 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43416 deadline: 1731751241523, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffe1934c280093c181c2b062a05e0908, server=037c58a19ddb,37267,1731751077844 2024-11-16T09:59:41,529 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731751162662.ffe1934c280093c181c2b062a05e0908., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffe1934c280093c181c2b062a05e0908, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:59:41,529 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731751162662.ffe1934c280093c181c2b062a05e0908., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 because the exception is null or not the one we care about 2024-11-16T09:59:41,529 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731751162662.ffe1934c280093c181c2b062a05e0908., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 , the old value is region=TestAcidGuarantees,,1731751162662.ffe1934c280093c181c2b062a05e0908., hostname=037c58a19ddb,37267,1731751077844, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffe1934c280093c181c2b062a05e0908, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:59:41,529 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731751162662.ffe1934c280093c181c2b062a05e0908., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffe1934c280093c181c2b062a05e0908, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:59:41,529 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731751162662.ffe1934c280093c181c2b062a05e0908., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 because the exception is null or not the one we care about 2024-11-16T09:59:41,539 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffe1934c280093c181c2b062a05e0908, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T09:59:41,539 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] ipc.CallRunner(138): callId: 80 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43362 deadline: 1731751241533, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffe1934c280093c181c2b062a05e0908, server=037c58a19ddb,37267,1731751077844 2024-11-16T09:59:41,540 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731751162662.ffe1934c280093c181c2b062a05e0908., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 , the old value is region=TestAcidGuarantees,,1731751162662.ffe1934c280093c181c2b062a05e0908., hostname=037c58a19ddb,37267,1731751077844, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffe1934c280093c181c2b062a05e0908, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:59:41,540 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731751162662.ffe1934c280093c181c2b062a05e0908., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffe1934c280093c181c2b062a05e0908, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:59:41,540 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731751162662.ffe1934c280093c181c2b062a05e0908., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 because the exception is null or not the one we care about 2024-11-16T09:59:41,563 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37877 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=105 2024-11-16T09:59:41,563 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-16T09:59:41,564 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37877 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-16T09:59:41,565 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37877 {}] procedure2.ProcedureExecutor(1139): Stored pid=107, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=107, table=TestAcidGuarantees 2024-11-16T09:59:41,566 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37877 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=107 2024-11-16T09:59:41,566 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=107, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=107, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-16T09:59:41,567 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=107, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=107, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-16T09:59:41,567 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=108, ppid=107, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-16T09:59:41,659 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=227, memsize=26.8 K, hasBloomFilter=true, into tmp file hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/ffe1934c280093c181c2b062a05e0908/.tmp/A/9e8350a74f8a4165b66b073a641f4577 2024-11-16T09:59:41,667 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/ffe1934c280093c181c2b062a05e0908/.tmp/B/d66400cdff5b4ff79a783a59ef3e2994 is 50, key is test_row_0/B:col10/1731751179682/Put/seqid=0 2024-11-16T09:59:41,670 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37591 is added to blk_1073742233_1409 (size=12151) 2024-11-16T09:59:41,671 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=227 (bloomFilter=true), to=hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/ffe1934c280093c181c2b062a05e0908/.tmp/B/d66400cdff5b4ff79a783a59ef3e2994 2024-11-16T09:59:41,673 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37877 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=107 2024-11-16T09:59:41,677 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/ffe1934c280093c181c2b062a05e0908/.tmp/C/715709ecd09c44af93fb2375ba1dbdbf is 50, key is test_row_0/C:col10/1731751179682/Put/seqid=0 2024-11-16T09:59:41,681 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37591 is added to blk_1073742234_1410 (size=12151) 2024-11-16T09:59:41,719 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=37267 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=108 2024-11-16T09:59:41,719 DEBUG [RS_FLUSH_OPERATIONS-regionserver/037c58a19ddb:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731751162662.ffe1934c280093c181c2b062a05e0908. 2024-11-16T09:59:41,719 DEBUG [RS_FLUSH_OPERATIONS-regionserver/037c58a19ddb:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731751162662.ffe1934c280093c181c2b062a05e0908. as already flushing 2024-11-16T09:59:41,719 DEBUG [RS_FLUSH_OPERATIONS-regionserver/037c58a19ddb:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731751162662.ffe1934c280093c181c2b062a05e0908. 2024-11-16T09:59:41,719 ERROR [RS_FLUSH_OPERATIONS-regionserver/037c58a19ddb:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] handler.RSProcedureHandler(58): pid=108 java.io.IOException: Unable to complete flush {ENCODED => ffe1934c280093c181c2b062a05e0908, NAME => 'TestAcidGuarantees,,1731751162662.ffe1934c280093c181c2b062a05e0908.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T09:59:41,719 DEBUG [RS_FLUSH_OPERATIONS-regionserver/037c58a19ddb:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=108 java.io.IOException: Unable to complete flush {ENCODED => ffe1934c280093c181c2b062a05e0908, NAME => 'TestAcidGuarantees,,1731751162662.ffe1934c280093c181c2b062a05e0908.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T09:59:41,720 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37877 {}] master.HMaster(4177): Remote procedure failed, pid=108 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ffe1934c280093c181c2b062a05e0908, NAME => 'TestAcidGuarantees,,1731751162662.ffe1934c280093c181c2b062a05e0908.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ffe1934c280093c181c2b062a05e0908, NAME => 'TestAcidGuarantees,,1731751162662.ffe1934c280093c181c2b062a05e0908.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T09:59:41,872 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=37267 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=108 2024-11-16T09:59:41,872 DEBUG [RS_FLUSH_OPERATIONS-regionserver/037c58a19ddb:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731751162662.ffe1934c280093c181c2b062a05e0908. 2024-11-16T09:59:41,872 DEBUG [RS_FLUSH_OPERATIONS-regionserver/037c58a19ddb:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731751162662.ffe1934c280093c181c2b062a05e0908. as already flushing 2024-11-16T09:59:41,872 DEBUG [RS_FLUSH_OPERATIONS-regionserver/037c58a19ddb:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731751162662.ffe1934c280093c181c2b062a05e0908. 2024-11-16T09:59:41,873 ERROR [RS_FLUSH_OPERATIONS-regionserver/037c58a19ddb:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] handler.RSProcedureHandler(58): pid=108 java.io.IOException: Unable to complete flush {ENCODED => ffe1934c280093c181c2b062a05e0908, NAME => 'TestAcidGuarantees,,1731751162662.ffe1934c280093c181c2b062a05e0908.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T09:59:41,873 DEBUG [RS_FLUSH_OPERATIONS-regionserver/037c58a19ddb:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=108 java.io.IOException: Unable to complete flush {ENCODED => ffe1934c280093c181c2b062a05e0908, NAME => 'TestAcidGuarantees,,1731751162662.ffe1934c280093c181c2b062a05e0908.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T09:59:41,873 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37877 {}] master.HMaster(4177): Remote procedure failed, pid=108 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ffe1934c280093c181c2b062a05e0908, NAME => 'TestAcidGuarantees,,1731751162662.ffe1934c280093c181c2b062a05e0908.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ffe1934c280093c181c2b062a05e0908, NAME => 'TestAcidGuarantees,,1731751162662.ffe1934c280093c181c2b062a05e0908.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T09:59:41,883 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37877 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=107 2024-11-16T09:59:42,026 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=37267 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=108 2024-11-16T09:59:42,026 DEBUG [RS_FLUSH_OPERATIONS-regionserver/037c58a19ddb:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731751162662.ffe1934c280093c181c2b062a05e0908. 2024-11-16T09:59:42,026 DEBUG [RS_FLUSH_OPERATIONS-regionserver/037c58a19ddb:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731751162662.ffe1934c280093c181c2b062a05e0908. as already flushing 2024-11-16T09:59:42,026 DEBUG [RS_FLUSH_OPERATIONS-regionserver/037c58a19ddb:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731751162662.ffe1934c280093c181c2b062a05e0908. 2024-11-16T09:59:42,026 ERROR [RS_FLUSH_OPERATIONS-regionserver/037c58a19ddb:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] handler.RSProcedureHandler(58): pid=108 java.io.IOException: Unable to complete flush {ENCODED => ffe1934c280093c181c2b062a05e0908, NAME => 'TestAcidGuarantees,,1731751162662.ffe1934c280093c181c2b062a05e0908.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T09:59:42,026 DEBUG [RS_FLUSH_OPERATIONS-regionserver/037c58a19ddb:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=108 java.io.IOException: Unable to complete flush {ENCODED => ffe1934c280093c181c2b062a05e0908, NAME => 'TestAcidGuarantees,,1731751162662.ffe1934c280093c181c2b062a05e0908.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T09:59:42,027 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37877 {}] master.HMaster(4177): Remote procedure failed, pid=108 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ffe1934c280093c181c2b062a05e0908, NAME => 'TestAcidGuarantees,,1731751162662.ffe1934c280093c181c2b062a05e0908.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ffe1934c280093c181c2b062a05e0908, NAME => 'TestAcidGuarantees,,1731751162662.ffe1934c280093c181c2b062a05e0908.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T09:59:42,030 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffe1934c280093c181c2b062a05e0908, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T09:59:42,030 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] ipc.CallRunner(138): callId: 87 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43418 deadline: 1731751242023, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffe1934c280093c181c2b062a05e0908, server=037c58a19ddb,37267,1731751077844 2024-11-16T09:59:42,031 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731751162662.ffe1934c280093c181c2b062a05e0908., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 , the old value is region=TestAcidGuarantees,,1731751162662.ffe1934c280093c181c2b062a05e0908., hostname=037c58a19ddb,37267,1731751077844, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffe1934c280093c181c2b062a05e0908, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:59:42,031 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731751162662.ffe1934c280093c181c2b062a05e0908., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffe1934c280093c181c2b062a05e0908, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:59:42,031 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731751162662.ffe1934c280093c181c2b062a05e0908., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 because the exception is null or not the one we care about 2024-11-16T09:59:42,041 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffe1934c280093c181c2b062a05e0908, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T09:59:42,041 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] ipc.CallRunner(138): callId: 85 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43332 deadline: 1731751242033, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffe1934c280093c181c2b062a05e0908, server=037c58a19ddb,37267,1731751077844 2024-11-16T09:59:42,041 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffe1934c280093c181c2b062a05e0908, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T09:59:42,041 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731751162662.ffe1934c280093c181c2b062a05e0908., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 , the old value is region=TestAcidGuarantees,,1731751162662.ffe1934c280093c181c2b062a05e0908., hostname=037c58a19ddb,37267,1731751077844, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffe1934c280093c181c2b062a05e0908, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:59:42,041 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] ipc.CallRunner(138): callId: 86 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43402 deadline: 1731751242033, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffe1934c280093c181c2b062a05e0908, server=037c58a19ddb,37267,1731751077844 2024-11-16T09:59:42,041 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731751162662.ffe1934c280093c181c2b062a05e0908., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffe1934c280093c181c2b062a05e0908, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:59:42,042 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731751162662.ffe1934c280093c181c2b062a05e0908., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 because the exception is null or not the one we care about 2024-11-16T09:59:42,042 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffe1934c280093c181c2b062a05e0908, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T09:59:42,042 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731751162662.ffe1934c280093c181c2b062a05e0908., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 , the old value is region=TestAcidGuarantees,,1731751162662.ffe1934c280093c181c2b062a05e0908., hostname=037c58a19ddb,37267,1731751077844, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffe1934c280093c181c2b062a05e0908, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:59:42,042 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731751162662.ffe1934c280093c181c2b062a05e0908., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffe1934c280093c181c2b062a05e0908, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:59:42,042 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] ipc.CallRunner(138): callId: 89 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43416 deadline: 1731751242033, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffe1934c280093c181c2b062a05e0908, server=037c58a19ddb,37267,1731751077844 2024-11-16T09:59:42,042 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731751162662.ffe1934c280093c181c2b062a05e0908., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 because the exception is null or not the one we care about 2024-11-16T09:59:42,042 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731751162662.ffe1934c280093c181c2b062a05e0908., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 , the old value is region=TestAcidGuarantees,,1731751162662.ffe1934c280093c181c2b062a05e0908., hostname=037c58a19ddb,37267,1731751077844, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffe1934c280093c181c2b062a05e0908, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:59:42,042 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731751162662.ffe1934c280093c181c2b062a05e0908., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffe1934c280093c181c2b062a05e0908, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:59:42,042 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731751162662.ffe1934c280093c181c2b062a05e0908., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 because the exception is null or not the one we care about 2024-11-16T09:59:42,044 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffe1934c280093c181c2b062a05e0908, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T09:59:42,044 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] ipc.CallRunner(138): callId: 81 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43362 deadline: 1731751242042, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffe1934c280093c181c2b062a05e0908, server=037c58a19ddb,37267,1731751077844 2024-11-16T09:59:42,045 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731751162662.ffe1934c280093c181c2b062a05e0908., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 , the old value is region=TestAcidGuarantees,,1731751162662.ffe1934c280093c181c2b062a05e0908., hostname=037c58a19ddb,37267,1731751077844, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffe1934c280093c181c2b062a05e0908, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:59:42,045 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731751162662.ffe1934c280093c181c2b062a05e0908., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffe1934c280093c181c2b062a05e0908, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:59:42,045 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731751162662.ffe1934c280093c181c2b062a05e0908., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 because the exception is null or not the one we care about 2024-11-16T09:59:42,082 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=227 (bloomFilter=true), to=hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/ffe1934c280093c181c2b062a05e0908/.tmp/C/715709ecd09c44af93fb2375ba1dbdbf 2024-11-16T09:59:42,087 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/ffe1934c280093c181c2b062a05e0908/.tmp/A/9e8350a74f8a4165b66b073a641f4577 as hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/ffe1934c280093c181c2b062a05e0908/A/9e8350a74f8a4165b66b073a641f4577 2024-11-16T09:59:42,091 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/ffe1934c280093c181c2b062a05e0908/A/9e8350a74f8a4165b66b073a641f4577, entries=200, sequenceid=227, filesize=38.8 K 2024-11-16T09:59:42,091 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/ffe1934c280093c181c2b062a05e0908/.tmp/B/d66400cdff5b4ff79a783a59ef3e2994 as hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/ffe1934c280093c181c2b062a05e0908/B/d66400cdff5b4ff79a783a59ef3e2994 2024-11-16T09:59:42,095 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/ffe1934c280093c181c2b062a05e0908/B/d66400cdff5b4ff79a783a59ef3e2994, entries=150, sequenceid=227, filesize=11.9 K 2024-11-16T09:59:42,096 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/ffe1934c280093c181c2b062a05e0908/.tmp/C/715709ecd09c44af93fb2375ba1dbdbf as hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/ffe1934c280093c181c2b062a05e0908/C/715709ecd09c44af93fb2375ba1dbdbf 2024-11-16T09:59:42,100 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/ffe1934c280093c181c2b062a05e0908/C/715709ecd09c44af93fb2375ba1dbdbf, entries=150, sequenceid=227, filesize=11.9 K 2024-11-16T09:59:42,101 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~80.51 KB/82440, heapSize ~211.64 KB/216720, currentSize=120.76 KB/123660 for ffe1934c280093c181c2b062a05e0908 in 1263ms, sequenceid=227, compaction requested=true 2024-11-16T09:59:42,101 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for ffe1934c280093c181c2b062a05e0908: 2024-11-16T09:59:42,101 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store ffe1934c280093c181c2b062a05e0908:A, priority=-2147483648, current under compaction store size is 1 2024-11-16T09:59:42,101 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-16T09:59:42,101 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store ffe1934c280093c181c2b062a05e0908:B, priority=-2147483648, current under compaction store size is 2 2024-11-16T09:59:42,101 DEBUG [RS:0;037c58a19ddb:37267-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-16T09:59:42,101 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-16T09:59:42,101 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store ffe1934c280093c181c2b062a05e0908:C, priority=-2147483648, current under compaction store size is 3 2024-11-16T09:59:42,101 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-16T09:59:42,101 DEBUG [RS:0;037c58a19ddb:37267-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-16T09:59:42,102 DEBUG [RS:0;037c58a19ddb:37267-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 102403 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-16T09:59:42,102 DEBUG [RS:0;037c58a19ddb:37267-shortCompactions-0 {}] regionserver.HStore(1541): ffe1934c280093c181c2b062a05e0908/A is initiating minor compaction (all files) 2024-11-16T09:59:42,102 DEBUG [RS:0;037c58a19ddb:37267-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36897 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-16T09:59:42,102 DEBUG [RS:0;037c58a19ddb:37267-longCompactions-0 {}] regionserver.HStore(1541): ffe1934c280093c181c2b062a05e0908/B is initiating minor compaction (all files) 2024-11-16T09:59:42,102 INFO [RS:0;037c58a19ddb:37267-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of ffe1934c280093c181c2b062a05e0908/A in TestAcidGuarantees,,1731751162662.ffe1934c280093c181c2b062a05e0908. 2024-11-16T09:59:42,102 INFO [RS:0;037c58a19ddb:37267-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of ffe1934c280093c181c2b062a05e0908/B in TestAcidGuarantees,,1731751162662.ffe1934c280093c181c2b062a05e0908. 2024-11-16T09:59:42,102 INFO [RS:0;037c58a19ddb:37267-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/ffe1934c280093c181c2b062a05e0908/B/32d1270366ac4a4688c1aae590fea7c5, hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/ffe1934c280093c181c2b062a05e0908/B/fa2fc42df2b14c9788de3b052407407a, hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/ffe1934c280093c181c2b062a05e0908/B/d66400cdff5b4ff79a783a59ef3e2994] into tmpdir=hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/ffe1934c280093c181c2b062a05e0908/.tmp, totalSize=36.0 K 2024-11-16T09:59:42,102 INFO [RS:0;037c58a19ddb:37267-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/ffe1934c280093c181c2b062a05e0908/A/3160dd6fd84e4b82b402bd46540cc5ed, hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/ffe1934c280093c181c2b062a05e0908/A/8e8ef152dca44dd8a3f29446df0b9790, hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/ffe1934c280093c181c2b062a05e0908/A/9e8350a74f8a4165b66b073a641f4577] into tmpdir=hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/ffe1934c280093c181c2b062a05e0908/.tmp, totalSize=100.0 K 2024-11-16T09:59:42,102 INFO [RS:0;037c58a19ddb:37267-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731751162662.ffe1934c280093c181c2b062a05e0908. 2024-11-16T09:59:42,102 DEBUG [RS:0;037c58a19ddb:37267-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731751162662.ffe1934c280093c181c2b062a05e0908. files: [hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/ffe1934c280093c181c2b062a05e0908/A/3160dd6fd84e4b82b402bd46540cc5ed, hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/ffe1934c280093c181c2b062a05e0908/A/8e8ef152dca44dd8a3f29446df0b9790, hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/ffe1934c280093c181c2b062a05e0908/A/9e8350a74f8a4165b66b073a641f4577] 2024-11-16T09:59:42,102 DEBUG [RS:0;037c58a19ddb:37267-longCompactions-0 {}] compactions.Compactor(225): Compacting 32d1270366ac4a4688c1aae590fea7c5, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=187, earliestPutTs=1731751176260 2024-11-16T09:59:42,103 DEBUG [RS:0;037c58a19ddb:37267-shortCompactions-0 {}] compactions.Compactor(225): Compacting 3160dd6fd84e4b82b402bd46540cc5ed, keycount=150, bloomtype=ROW, size=30.8 K, encoding=NONE, compression=NONE, seqNum=187, earliestPutTs=1731751176260 2024-11-16T09:59:42,103 DEBUG [RS:0;037c58a19ddb:37267-longCompactions-0 {}] compactions.Compactor(225): Compacting fa2fc42df2b14c9788de3b052407407a, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=209, earliestPutTs=1731751177483 2024-11-16T09:59:42,103 DEBUG [RS:0;037c58a19ddb:37267-shortCompactions-0 {}] compactions.Compactor(225): Compacting 8e8ef152dca44dd8a3f29446df0b9790, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=209, earliestPutTs=1731751177483 2024-11-16T09:59:42,103 DEBUG [RS:0;037c58a19ddb:37267-longCompactions-0 {}] compactions.Compactor(225): Compacting d66400cdff5b4ff79a783a59ef3e2994, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=227, earliestPutTs=1731751179681 2024-11-16T09:59:42,103 DEBUG [RS:0;037c58a19ddb:37267-shortCompactions-0 {}] compactions.Compactor(225): Compacting 9e8350a74f8a4165b66b073a641f4577, keycount=200, bloomtype=ROW, size=38.8 K, encoding=NONE, compression=NONE, seqNum=227, earliestPutTs=1731751179675 2024-11-16T09:59:42,111 INFO [RS:0;037c58a19ddb:37267-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): ffe1934c280093c181c2b062a05e0908#B#compaction#347 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-16T09:59:42,112 DEBUG [RS:0;037c58a19ddb:37267-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/ffe1934c280093c181c2b062a05e0908/.tmp/B/04b0e51d058b49c7ac39bd0526239b72 is 50, key is test_row_0/B:col10/1731751179682/Put/seqid=0 2024-11-16T09:59:42,124 INFO [RS:0;037c58a19ddb:37267-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=ffe1934c280093c181c2b062a05e0908] 2024-11-16T09:59:42,128 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37591 is added to blk_1073742235_1411 (size=12697) 2024-11-16T09:59:42,128 DEBUG [RS:0;037c58a19ddb:37267-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e2024111676bd01c00e0c45f9b31940fd7970c565_ffe1934c280093c181c2b062a05e0908 store=[table=TestAcidGuarantees family=A region=ffe1934c280093c181c2b062a05e0908] 2024-11-16T09:59:42,130 DEBUG [RS:0;037c58a19ddb:37267-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e2024111676bd01c00e0c45f9b31940fd7970c565_ffe1934c280093c181c2b062a05e0908, store=[table=TestAcidGuarantees family=A region=ffe1934c280093c181c2b062a05e0908] 2024-11-16T09:59:42,130 DEBUG [RS:0;037c58a19ddb:37267-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111676bd01c00e0c45f9b31940fd7970c565_ffe1934c280093c181c2b062a05e0908 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=ffe1934c280093c181c2b062a05e0908] 2024-11-16T09:59:42,137 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37591 is added to blk_1073742236_1412 (size=4469) 2024-11-16T09:59:42,179 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=37267 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=108 2024-11-16T09:59:42,179 DEBUG [RS_FLUSH_OPERATIONS-regionserver/037c58a19ddb:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731751162662.ffe1934c280093c181c2b062a05e0908. 2024-11-16T09:59:42,179 INFO [RS_FLUSH_OPERATIONS-regionserver/037c58a19ddb:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HRegion(2902): Flushing ffe1934c280093c181c2b062a05e0908 3/3 column families, dataSize=120.76 KB heapSize=317.16 KB 2024-11-16T09:59:42,186 DEBUG [RS_FLUSH_OPERATIONS-regionserver/037c58a19ddb:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111676c0b879531f4f18a7ce5c7bdde3a78c_ffe1934c280093c181c2b062a05e0908 is 50, key is test_row_0/A:col10/1731751180879/Put/seqid=0 2024-11-16T09:59:42,190 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37591 is added to blk_1073742237_1413 (size=12304) 2024-11-16T09:59:42,193 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37877 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=107 2024-11-16T09:59:42,533 DEBUG [RS:0;037c58a19ddb:37267-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/ffe1934c280093c181c2b062a05e0908/.tmp/B/04b0e51d058b49c7ac39bd0526239b72 as hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/ffe1934c280093c181c2b062a05e0908/B/04b0e51d058b49c7ac39bd0526239b72 2024-11-16T09:59:42,537 INFO [RS:0;037c58a19ddb:37267-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in ffe1934c280093c181c2b062a05e0908/B of ffe1934c280093c181c2b062a05e0908 into 04b0e51d058b49c7ac39bd0526239b72(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-16T09:59:42,537 DEBUG [RS:0;037c58a19ddb:37267-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for ffe1934c280093c181c2b062a05e0908: 2024-11-16T09:59:42,537 INFO [RS:0;037c58a19ddb:37267-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731751162662.ffe1934c280093c181c2b062a05e0908., storeName=ffe1934c280093c181c2b062a05e0908/B, priority=13, startTime=1731751182101; duration=0sec 2024-11-16T09:59:42,537 DEBUG [RS:0;037c58a19ddb:37267-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-16T09:59:42,538 DEBUG [RS:0;037c58a19ddb:37267-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: ffe1934c280093c181c2b062a05e0908:B 2024-11-16T09:59:42,538 DEBUG [RS:0;037c58a19ddb:37267-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-16T09:59:42,538 INFO [RS:0;037c58a19ddb:37267-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): ffe1934c280093c181c2b062a05e0908#A#compaction#348 average throughput is 0.06 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-16T09:59:42,539 DEBUG [RS:0;037c58a19ddb:37267-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/ffe1934c280093c181c2b062a05e0908/.tmp/A/2bbc77228e7446d4826467c6853a1981 is 175, key is test_row_0/A:col10/1731751179682/Put/seqid=0 2024-11-16T09:59:42,540 DEBUG [RS:0;037c58a19ddb:37267-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36897 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-16T09:59:42,540 DEBUG [RS:0;037c58a19ddb:37267-longCompactions-0 {}] regionserver.HStore(1541): ffe1934c280093c181c2b062a05e0908/C is initiating minor compaction (all files) 2024-11-16T09:59:42,540 INFO [RS:0;037c58a19ddb:37267-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of ffe1934c280093c181c2b062a05e0908/C in TestAcidGuarantees,,1731751162662.ffe1934c280093c181c2b062a05e0908. 2024-11-16T09:59:42,540 INFO [RS:0;037c58a19ddb:37267-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/ffe1934c280093c181c2b062a05e0908/C/5ec075b689da412bb397e9b9dbafdfe3, hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/ffe1934c280093c181c2b062a05e0908/C/a0afb7c2bda64ca680e563efa5618949, hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/ffe1934c280093c181c2b062a05e0908/C/715709ecd09c44af93fb2375ba1dbdbf] into tmpdir=hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/ffe1934c280093c181c2b062a05e0908/.tmp, totalSize=36.0 K 2024-11-16T09:59:42,541 DEBUG [RS:0;037c58a19ddb:37267-longCompactions-0 {}] compactions.Compactor(225): Compacting 5ec075b689da412bb397e9b9dbafdfe3, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=187, earliestPutTs=1731751176260 2024-11-16T09:59:42,541 DEBUG [RS:0;037c58a19ddb:37267-longCompactions-0 {}] compactions.Compactor(225): Compacting a0afb7c2bda64ca680e563efa5618949, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=209, earliestPutTs=1731751177483 2024-11-16T09:59:42,541 DEBUG [RS:0;037c58a19ddb:37267-longCompactions-0 {}] compactions.Compactor(225): Compacting 715709ecd09c44af93fb2375ba1dbdbf, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=227, earliestPutTs=1731751179681 2024-11-16T09:59:42,549 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37591 is added to blk_1073742238_1414 (size=31651) 2024-11-16T09:59:42,554 INFO [RS:0;037c58a19ddb:37267-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): ffe1934c280093c181c2b062a05e0908#C#compaction#350 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-16T09:59:42,554 DEBUG [RS:0;037c58a19ddb:37267-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/ffe1934c280093c181c2b062a05e0908/.tmp/C/c498d0fe177440029170dc63c55c9c94 is 50, key is test_row_0/C:col10/1731751179682/Put/seqid=0 2024-11-16T09:59:42,557 DEBUG [RS:0;037c58a19ddb:37267-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/ffe1934c280093c181c2b062a05e0908/.tmp/A/2bbc77228e7446d4826467c6853a1981 as hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/ffe1934c280093c181c2b062a05e0908/A/2bbc77228e7446d4826467c6853a1981 2024-11-16T09:59:42,561 INFO [RS:0;037c58a19ddb:37267-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in ffe1934c280093c181c2b062a05e0908/A of ffe1934c280093c181c2b062a05e0908 into 2bbc77228e7446d4826467c6853a1981(size=30.9 K), total size for store is 30.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-16T09:59:42,561 DEBUG [RS:0;037c58a19ddb:37267-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for ffe1934c280093c181c2b062a05e0908: 2024-11-16T09:59:42,561 INFO [RS:0;037c58a19ddb:37267-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731751162662.ffe1934c280093c181c2b062a05e0908., storeName=ffe1934c280093c181c2b062a05e0908/A, priority=13, startTime=1731751182101; duration=0sec 2024-11-16T09:59:42,561 DEBUG [RS:0;037c58a19ddb:37267-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-16T09:59:42,561 DEBUG [RS:0;037c58a19ddb:37267-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: ffe1934c280093c181c2b062a05e0908:A 2024-11-16T09:59:42,566 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37591 is added to blk_1073742239_1415 (size=12697) 2024-11-16T09:59:42,571 DEBUG [RS:0;037c58a19ddb:37267-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/ffe1934c280093c181c2b062a05e0908/.tmp/C/c498d0fe177440029170dc63c55c9c94 as hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/ffe1934c280093c181c2b062a05e0908/C/c498d0fe177440029170dc63c55c9c94 2024-11-16T09:59:42,575 INFO [RS:0;037c58a19ddb:37267-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in ffe1934c280093c181c2b062a05e0908/C of ffe1934c280093c181c2b062a05e0908 into c498d0fe177440029170dc63c55c9c94(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-16T09:59:42,575 DEBUG [RS:0;037c58a19ddb:37267-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for ffe1934c280093c181c2b062a05e0908: 2024-11-16T09:59:42,575 INFO [RS:0;037c58a19ddb:37267-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731751162662.ffe1934c280093c181c2b062a05e0908., storeName=ffe1934c280093c181c2b062a05e0908/C, priority=13, startTime=1731751182101; duration=0sec 2024-11-16T09:59:42,575 DEBUG [RS:0;037c58a19ddb:37267-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-16T09:59:42,575 DEBUG [RS:0;037c58a19ddb:37267-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: ffe1934c280093c181c2b062a05e0908:C 2024-11-16T09:59:42,590 DEBUG [RS_FLUSH_OPERATIONS-regionserver/037c58a19ddb:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T09:59:42,593 INFO [RS_FLUSH_OPERATIONS-regionserver/037c58a19ddb:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111676c0b879531f4f18a7ce5c7bdde3a78c_ffe1934c280093c181c2b062a05e0908 to hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111676c0b879531f4f18a7ce5c7bdde3a78c_ffe1934c280093c181c2b062a05e0908 2024-11-16T09:59:42,594 DEBUG [RS_FLUSH_OPERATIONS-regionserver/037c58a19ddb:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/ffe1934c280093c181c2b062a05e0908/.tmp/A/dac023d72c9a467cb301eb07f30695ca, store: [table=TestAcidGuarantees family=A region=ffe1934c280093c181c2b062a05e0908] 2024-11-16T09:59:42,595 DEBUG [RS_FLUSH_OPERATIONS-regionserver/037c58a19ddb:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/ffe1934c280093c181c2b062a05e0908/.tmp/A/dac023d72c9a467cb301eb07f30695ca is 175, key is test_row_0/A:col10/1731751180879/Put/seqid=0 2024-11-16T09:59:42,598 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37591 is added to blk_1073742240_1416 (size=31105) 2024-11-16T09:59:42,702 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37877 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=107 2024-11-16T09:59:42,999 INFO [RS_FLUSH_OPERATIONS-regionserver/037c58a19ddb:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=248, memsize=40.3 K, hasBloomFilter=true, into tmp file hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/ffe1934c280093c181c2b062a05e0908/.tmp/A/dac023d72c9a467cb301eb07f30695ca 2024-11-16T09:59:43,007 DEBUG [RS_FLUSH_OPERATIONS-regionserver/037c58a19ddb:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/ffe1934c280093c181c2b062a05e0908/.tmp/B/dc55e610407e49158964b1ae68c11f86 is 50, key is test_row_0/B:col10/1731751180879/Put/seqid=0 2024-11-16T09:59:43,011 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37591 is added to blk_1073742241_1417 (size=12151) 2024-11-16T09:59:43,047 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] regionserver.HRegion(8855): Flush requested on ffe1934c280093c181c2b062a05e0908 2024-11-16T09:59:43,047 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731751162662.ffe1934c280093c181c2b062a05e0908. as already flushing 2024-11-16T09:59:43,071 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffe1934c280093c181c2b062a05e0908, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T09:59:43,071 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] ipc.CallRunner(138): callId: 88 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43332 deadline: 1731751243068, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffe1934c280093c181c2b062a05e0908, server=037c58a19ddb,37267,1731751077844 2024-11-16T09:59:43,071 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731751162662.ffe1934c280093c181c2b062a05e0908., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 , the old value is region=TestAcidGuarantees,,1731751162662.ffe1934c280093c181c2b062a05e0908., hostname=037c58a19ddb,37267,1731751077844, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffe1934c280093c181c2b062a05e0908, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:59:43,071 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731751162662.ffe1934c280093c181c2b062a05e0908., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffe1934c280093c181c2b062a05e0908, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:59:43,071 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731751162662.ffe1934c280093c181c2b062a05e0908., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 because the exception is null or not the one we care about 2024-11-16T09:59:43,078 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffe1934c280093c181c2b062a05e0908, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T09:59:43,078 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] ipc.CallRunner(138): callId: 89 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43402 deadline: 1731751243069, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffe1934c280093c181c2b062a05e0908, server=037c58a19ddb,37267,1731751077844 2024-11-16T09:59:43,078 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731751162662.ffe1934c280093c181c2b062a05e0908., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 , the old value is region=TestAcidGuarantees,,1731751162662.ffe1934c280093c181c2b062a05e0908., hostname=037c58a19ddb,37267,1731751077844, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffe1934c280093c181c2b062a05e0908, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:59:43,078 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731751162662.ffe1934c280093c181c2b062a05e0908., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffe1934c280093c181c2b062a05e0908, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:59:43,079 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731751162662.ffe1934c280093c181c2b062a05e0908., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 because the exception is null or not the one we care about 2024-11-16T09:59:43,079 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffe1934c280093c181c2b062a05e0908, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T09:59:43,080 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] ipc.CallRunner(138): callId: 92 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43416 deadline: 1731751243069, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffe1934c280093c181c2b062a05e0908, server=037c58a19ddb,37267,1731751077844 2024-11-16T09:59:43,080 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffe1934c280093c181c2b062a05e0908, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T09:59:43,080 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731751162662.ffe1934c280093c181c2b062a05e0908., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 , the old value is region=TestAcidGuarantees,,1731751162662.ffe1934c280093c181c2b062a05e0908., hostname=037c58a19ddb,37267,1731751077844, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffe1934c280093c181c2b062a05e0908, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:59:43,080 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731751162662.ffe1934c280093c181c2b062a05e0908., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffe1934c280093c181c2b062a05e0908, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:59:43,080 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731751162662.ffe1934c280093c181c2b062a05e0908., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 because the exception is null or not the one we care about 2024-11-16T09:59:43,080 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] ipc.CallRunner(138): callId: 84 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43362 deadline: 1731751243070, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffe1934c280093c181c2b062a05e0908, server=037c58a19ddb,37267,1731751077844 2024-11-16T09:59:43,080 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731751162662.ffe1934c280093c181c2b062a05e0908., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 , the old value is region=TestAcidGuarantees,,1731751162662.ffe1934c280093c181c2b062a05e0908., hostname=037c58a19ddb,37267,1731751077844, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffe1934c280093c181c2b062a05e0908, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:59:43,080 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731751162662.ffe1934c280093c181c2b062a05e0908., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffe1934c280093c181c2b062a05e0908, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:59:43,080 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffe1934c280093c181c2b062a05e0908, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T09:59:43,080 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731751162662.ffe1934c280093c181c2b062a05e0908., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 because the exception is null or not the one we care about 2024-11-16T09:59:43,080 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] ipc.CallRunner(138): callId: 92 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43418 deadline: 1731751243071, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffe1934c280093c181c2b062a05e0908, server=037c58a19ddb,37267,1731751077844 2024-11-16T09:59:43,081 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731751162662.ffe1934c280093c181c2b062a05e0908., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 , the old value is region=TestAcidGuarantees,,1731751162662.ffe1934c280093c181c2b062a05e0908., hostname=037c58a19ddb,37267,1731751077844, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffe1934c280093c181c2b062a05e0908, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:59:43,081 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731751162662.ffe1934c280093c181c2b062a05e0908., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffe1934c280093c181c2b062a05e0908, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:59:43,081 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731751162662.ffe1934c280093c181c2b062a05e0908., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 because the exception is null or not the one we care about 2024-11-16T09:59:43,177 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffe1934c280093c181c2b062a05e0908, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T09:59:43,177 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] ipc.CallRunner(138): callId: 89 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43332 deadline: 1731751243172, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffe1934c280093c181c2b062a05e0908, server=037c58a19ddb,37267,1731751077844 2024-11-16T09:59:43,178 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731751162662.ffe1934c280093c181c2b062a05e0908., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 , the old value is region=TestAcidGuarantees,,1731751162662.ffe1934c280093c181c2b062a05e0908., hostname=037c58a19ddb,37267,1731751077844, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffe1934c280093c181c2b062a05e0908, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:59:43,178 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731751162662.ffe1934c280093c181c2b062a05e0908., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffe1934c280093c181c2b062a05e0908, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:59:43,178 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731751162662.ffe1934c280093c181c2b062a05e0908., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 because the exception is null or not the one we care about 2024-11-16T09:59:43,188 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffe1934c280093c181c2b062a05e0908, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T09:59:43,188 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] ipc.CallRunner(138): callId: 90 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43402 deadline: 1731751243183, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffe1934c280093c181c2b062a05e0908, server=037c58a19ddb,37267,1731751077844 2024-11-16T09:59:43,188 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffe1934c280093c181c2b062a05e0908, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T09:59:43,188 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] ipc.CallRunner(138): callId: 93 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43416 deadline: 1731751243183, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffe1934c280093c181c2b062a05e0908, server=037c58a19ddb,37267,1731751077844 2024-11-16T09:59:43,188 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731751162662.ffe1934c280093c181c2b062a05e0908., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 , the old value is region=TestAcidGuarantees,,1731751162662.ffe1934c280093c181c2b062a05e0908., hostname=037c58a19ddb,37267,1731751077844, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffe1934c280093c181c2b062a05e0908, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:59:43,188 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731751162662.ffe1934c280093c181c2b062a05e0908., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffe1934c280093c181c2b062a05e0908, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:59:43,188 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731751162662.ffe1934c280093c181c2b062a05e0908., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 because the exception is null or not the one we care about 2024-11-16T09:59:43,188 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffe1934c280093c181c2b062a05e0908, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T09:59:43,188 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731751162662.ffe1934c280093c181c2b062a05e0908., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 , the old value is region=TestAcidGuarantees,,1731751162662.ffe1934c280093c181c2b062a05e0908., hostname=037c58a19ddb,37267,1731751077844, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffe1934c280093c181c2b062a05e0908, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:59:43,188 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] ipc.CallRunner(138): callId: 93 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43418 deadline: 1731751243183, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffe1934c280093c181c2b062a05e0908, server=037c58a19ddb,37267,1731751077844 2024-11-16T09:59:43,188 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731751162662.ffe1934c280093c181c2b062a05e0908., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffe1934c280093c181c2b062a05e0908, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:59:43,188 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731751162662.ffe1934c280093c181c2b062a05e0908., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 because the exception is null or not the one we care about 2024-11-16T09:59:43,188 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffe1934c280093c181c2b062a05e0908, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T09:59:43,188 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731751162662.ffe1934c280093c181c2b062a05e0908., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 , the old value is region=TestAcidGuarantees,,1731751162662.ffe1934c280093c181c2b062a05e0908., hostname=037c58a19ddb,37267,1731751077844, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffe1934c280093c181c2b062a05e0908, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:59:43,189 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] ipc.CallRunner(138): callId: 85 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43362 deadline: 1731751243183, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffe1934c280093c181c2b062a05e0908, server=037c58a19ddb,37267,1731751077844 2024-11-16T09:59:43,189 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731751162662.ffe1934c280093c181c2b062a05e0908., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffe1934c280093c181c2b062a05e0908, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:59:43,189 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731751162662.ffe1934c280093c181c2b062a05e0908., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 because the exception is null or not the one we care about 2024-11-16T09:59:43,189 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731751162662.ffe1934c280093c181c2b062a05e0908., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 , the old value is region=TestAcidGuarantees,,1731751162662.ffe1934c280093c181c2b062a05e0908., hostname=037c58a19ddb,37267,1731751077844, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffe1934c280093c181c2b062a05e0908, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:59:43,189 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731751162662.ffe1934c280093c181c2b062a05e0908., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffe1934c280093c181c2b062a05e0908, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:59:43,189 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731751162662.ffe1934c280093c181c2b062a05e0908., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 because the exception is null or not the one we care about 2024-11-16T09:59:43,390 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffe1934c280093c181c2b062a05e0908, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T09:59:43,390 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] ipc.CallRunner(138): callId: 90 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43332 deadline: 1731751243383, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffe1934c280093c181c2b062a05e0908, server=037c58a19ddb,37267,1731751077844 2024-11-16T09:59:43,390 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731751162662.ffe1934c280093c181c2b062a05e0908., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 , the old value is region=TestAcidGuarantees,,1731751162662.ffe1934c280093c181c2b062a05e0908., hostname=037c58a19ddb,37267,1731751077844, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffe1934c280093c181c2b062a05e0908, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:59:43,390 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731751162662.ffe1934c280093c181c2b062a05e0908., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffe1934c280093c181c2b062a05e0908, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:59:43,390 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731751162662.ffe1934c280093c181c2b062a05e0908., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 because the exception is null or not the one we care about 2024-11-16T09:59:43,400 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffe1934c280093c181c2b062a05e0908, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T09:59:43,400 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] ipc.CallRunner(138): callId: 91 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43402 deadline: 1731751243393, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffe1934c280093c181c2b062a05e0908, server=037c58a19ddb,37267,1731751077844 2024-11-16T09:59:43,400 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731751162662.ffe1934c280093c181c2b062a05e0908., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 , the old value is region=TestAcidGuarantees,,1731751162662.ffe1934c280093c181c2b062a05e0908., hostname=037c58a19ddb,37267,1731751077844, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffe1934c280093c181c2b062a05e0908, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:59:43,400 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731751162662.ffe1934c280093c181c2b062a05e0908., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffe1934c280093c181c2b062a05e0908, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:59:43,400 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731751162662.ffe1934c280093c181c2b062a05e0908., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 because the exception is null or not the one we care about 2024-11-16T09:59:43,401 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffe1934c280093c181c2b062a05e0908, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T09:59:43,401 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] ipc.CallRunner(138): callId: 94 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43416 deadline: 1731751243393, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffe1934c280093c181c2b062a05e0908, server=037c58a19ddb,37267,1731751077844 2024-11-16T09:59:43,402 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731751162662.ffe1934c280093c181c2b062a05e0908., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 , the old value is region=TestAcidGuarantees,,1731751162662.ffe1934c280093c181c2b062a05e0908., hostname=037c58a19ddb,37267,1731751077844, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffe1934c280093c181c2b062a05e0908, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:59:43,402 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731751162662.ffe1934c280093c181c2b062a05e0908., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffe1934c280093c181c2b062a05e0908, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:59:43,402 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffe1934c280093c181c2b062a05e0908, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T09:59:43,402 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731751162662.ffe1934c280093c181c2b062a05e0908., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 because the exception is null or not the one we care about 2024-11-16T09:59:43,402 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] ipc.CallRunner(138): callId: 94 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43418 deadline: 1731751243393, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffe1934c280093c181c2b062a05e0908, server=037c58a19ddb,37267,1731751077844 2024-11-16T09:59:43,402 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731751162662.ffe1934c280093c181c2b062a05e0908., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 , the old value is region=TestAcidGuarantees,,1731751162662.ffe1934c280093c181c2b062a05e0908., hostname=037c58a19ddb,37267,1731751077844, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffe1934c280093c181c2b062a05e0908, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:59:43,402 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731751162662.ffe1934c280093c181c2b062a05e0908., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffe1934c280093c181c2b062a05e0908, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:59:43,402 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731751162662.ffe1934c280093c181c2b062a05e0908., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 because the exception is null or not the one we care about 2024-11-16T09:59:43,402 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffe1934c280093c181c2b062a05e0908, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T09:59:43,402 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] ipc.CallRunner(138): callId: 86 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43362 deadline: 1731751243393, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffe1934c280093c181c2b062a05e0908, server=037c58a19ddb,37267,1731751077844 2024-11-16T09:59:43,402 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731751162662.ffe1934c280093c181c2b062a05e0908., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 , the old value is region=TestAcidGuarantees,,1731751162662.ffe1934c280093c181c2b062a05e0908., hostname=037c58a19ddb,37267,1731751077844, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffe1934c280093c181c2b062a05e0908, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:59:43,402 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731751162662.ffe1934c280093c181c2b062a05e0908., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffe1934c280093c181c2b062a05e0908, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:59:43,402 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731751162662.ffe1934c280093c181c2b062a05e0908., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 because the exception is null or not the one we care about 2024-11-16T09:59:43,412 INFO [RS_FLUSH_OPERATIONS-regionserver/037c58a19ddb:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=248 (bloomFilter=true), to=hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/ffe1934c280093c181c2b062a05e0908/.tmp/B/dc55e610407e49158964b1ae68c11f86 2024-11-16T09:59:43,419 DEBUG [RS_FLUSH_OPERATIONS-regionserver/037c58a19ddb:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/ffe1934c280093c181c2b062a05e0908/.tmp/C/0f4fae61ac1e4e00bebc2ddf2341d072 is 50, key is test_row_0/C:col10/1731751180879/Put/seqid=0 2024-11-16T09:59:43,424 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37591 is added to blk_1073742242_1418 (size=12151) 2024-11-16T09:59:43,424 INFO [RS_FLUSH_OPERATIONS-regionserver/037c58a19ddb:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=248 (bloomFilter=true), to=hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/ffe1934c280093c181c2b062a05e0908/.tmp/C/0f4fae61ac1e4e00bebc2ddf2341d072 2024-11-16T09:59:43,429 DEBUG [RS_FLUSH_OPERATIONS-regionserver/037c58a19ddb:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/ffe1934c280093c181c2b062a05e0908/.tmp/A/dac023d72c9a467cb301eb07f30695ca as hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/ffe1934c280093c181c2b062a05e0908/A/dac023d72c9a467cb301eb07f30695ca 2024-11-16T09:59:43,434 INFO [RS_FLUSH_OPERATIONS-regionserver/037c58a19ddb:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/ffe1934c280093c181c2b062a05e0908/A/dac023d72c9a467cb301eb07f30695ca, entries=150, sequenceid=248, filesize=30.4 K 2024-11-16T09:59:43,435 DEBUG [RS_FLUSH_OPERATIONS-regionserver/037c58a19ddb:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/ffe1934c280093c181c2b062a05e0908/.tmp/B/dc55e610407e49158964b1ae68c11f86 as hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/ffe1934c280093c181c2b062a05e0908/B/dc55e610407e49158964b1ae68c11f86 2024-11-16T09:59:43,438 INFO [RS_FLUSH_OPERATIONS-regionserver/037c58a19ddb:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/ffe1934c280093c181c2b062a05e0908/B/dc55e610407e49158964b1ae68c11f86, entries=150, sequenceid=248, filesize=11.9 K 2024-11-16T09:59:43,439 DEBUG [RS_FLUSH_OPERATIONS-regionserver/037c58a19ddb:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/ffe1934c280093c181c2b062a05e0908/.tmp/C/0f4fae61ac1e4e00bebc2ddf2341d072 as hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/ffe1934c280093c181c2b062a05e0908/C/0f4fae61ac1e4e00bebc2ddf2341d072 2024-11-16T09:59:43,443 INFO [RS_FLUSH_OPERATIONS-regionserver/037c58a19ddb:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/ffe1934c280093c181c2b062a05e0908/C/0f4fae61ac1e4e00bebc2ddf2341d072, entries=150, sequenceid=248, filesize=11.9 K 2024-11-16T09:59:43,444 INFO [RS_FLUSH_OPERATIONS-regionserver/037c58a19ddb:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HRegion(3140): Finished flush of dataSize ~120.76 KB/123660, heapSize ~317.11 KB/324720, currentSize=80.51 KB/82440 for ffe1934c280093c181c2b062a05e0908 in 1265ms, sequenceid=248, compaction requested=false 2024-11-16T09:59:43,444 DEBUG [RS_FLUSH_OPERATIONS-regionserver/037c58a19ddb:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HRegion(2603): Flush status journal for ffe1934c280093c181c2b062a05e0908: 2024-11-16T09:59:43,444 DEBUG [RS_FLUSH_OPERATIONS-regionserver/037c58a19ddb:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731751162662.ffe1934c280093c181c2b062a05e0908. 2024-11-16T09:59:43,444 DEBUG [RS_FLUSH_OPERATIONS-regionserver/037c58a19ddb:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=108 2024-11-16T09:59:43,445 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37877 {}] master.HMaster(4169): Remote procedure done, pid=108 2024-11-16T09:59:43,447 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=108, resume processing ppid=107 2024-11-16T09:59:43,447 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=108, ppid=107, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.8780 sec 2024-11-16T09:59:43,449 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=107, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=107, table=TestAcidGuarantees in 1.8830 sec 2024-11-16T09:59:43,704 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] regionserver.HRegion(8855): Flush requested on ffe1934c280093c181c2b062a05e0908 2024-11-16T09:59:43,704 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-16T09:59:43,705 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing ffe1934c280093c181c2b062a05e0908 3/3 column families, dataSize=87.22 KB heapSize=229.27 KB 2024-11-16T09:59:43,711 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111656efabdd34124ec596e7ad480ca80c50_ffe1934c280093c181c2b062a05e0908 is 50, key is test_row_0/A:col10/1731751183703/Put/seqid=0 2024-11-16T09:59:43,713 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37877 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=107 2024-11-16T09:59:43,713 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-16T09:59:43,714 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37877 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-16T09:59:43,715 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37877 {}] procedure2.ProcedureExecutor(1139): Stored pid=109, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=109, table=TestAcidGuarantees 2024-11-16T09:59:43,716 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37877 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=109 2024-11-16T09:59:43,717 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=109, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=109, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-16T09:59:43,717 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37591 is added to blk_1073742243_1419 (size=14994) 2024-11-16T09:59:43,717 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=109, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=109, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-16T09:59:43,717 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=110, ppid=109, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-16T09:59:43,752 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffe1934c280093c181c2b062a05e0908, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T09:59:43,752 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] ipc.CallRunner(138): callId: 95 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43332 deadline: 1731751243734, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffe1934c280093c181c2b062a05e0908, server=037c58a19ddb,37267,1731751077844 2024-11-16T09:59:43,752 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffe1934c280093c181c2b062a05e0908, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T09:59:43,752 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731751162662.ffe1934c280093c181c2b062a05e0908., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 , the old value is region=TestAcidGuarantees,,1731751162662.ffe1934c280093c181c2b062a05e0908., hostname=037c58a19ddb,37267,1731751077844, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffe1934c280093c181c2b062a05e0908, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:59:43,752 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731751162662.ffe1934c280093c181c2b062a05e0908., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffe1934c280093c181c2b062a05e0908, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:59:43,752 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] ipc.CallRunner(138): callId: 98 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43416 deadline: 1731751243735, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffe1934c280093c181c2b062a05e0908, server=037c58a19ddb,37267,1731751077844 2024-11-16T09:59:43,753 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731751162662.ffe1934c280093c181c2b062a05e0908., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 because the exception is null or not the one we care about 2024-11-16T09:59:43,753 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731751162662.ffe1934c280093c181c2b062a05e0908., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 , the old value is region=TestAcidGuarantees,,1731751162662.ffe1934c280093c181c2b062a05e0908., hostname=037c58a19ddb,37267,1731751077844, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffe1934c280093c181c2b062a05e0908, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:59:43,753 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731751162662.ffe1934c280093c181c2b062a05e0908., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffe1934c280093c181c2b062a05e0908, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:59:43,753 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffe1934c280093c181c2b062a05e0908, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T09:59:43,753 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731751162662.ffe1934c280093c181c2b062a05e0908., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 because the exception is null or not the one we care about 2024-11-16T09:59:43,753 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] ipc.CallRunner(138): callId: 98 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43418 deadline: 1731751243736, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffe1934c280093c181c2b062a05e0908, server=037c58a19ddb,37267,1731751077844 2024-11-16T09:59:43,753 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731751162662.ffe1934c280093c181c2b062a05e0908., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 , the old value is region=TestAcidGuarantees,,1731751162662.ffe1934c280093c181c2b062a05e0908., hostname=037c58a19ddb,37267,1731751077844, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffe1934c280093c181c2b062a05e0908, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:59:43,753 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731751162662.ffe1934c280093c181c2b062a05e0908., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffe1934c280093c181c2b062a05e0908, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:59:43,753 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731751162662.ffe1934c280093c181c2b062a05e0908., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 because the exception is null or not the one we care about 2024-11-16T09:59:43,759 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffe1934c280093c181c2b062a05e0908, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T09:59:43,759 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] ipc.CallRunner(138): callId: 90 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43362 deadline: 1731751243751, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffe1934c280093c181c2b062a05e0908, server=037c58a19ddb,37267,1731751077844 2024-11-16T09:59:43,759 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731751162662.ffe1934c280093c181c2b062a05e0908., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 , the old value is region=TestAcidGuarantees,,1731751162662.ffe1934c280093c181c2b062a05e0908., hostname=037c58a19ddb,37267,1731751077844, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffe1934c280093c181c2b062a05e0908, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:59:43,759 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731751162662.ffe1934c280093c181c2b062a05e0908., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffe1934c280093c181c2b062a05e0908, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:59:43,759 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731751162662.ffe1934c280093c181c2b062a05e0908., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 because the exception is null or not the one we care about 2024-11-16T09:59:43,762 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffe1934c280093c181c2b062a05e0908, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T09:59:43,762 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] ipc.CallRunner(138): callId: 97 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43402 deadline: 1731751243752, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffe1934c280093c181c2b062a05e0908, server=037c58a19ddb,37267,1731751077844 2024-11-16T09:59:43,762 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731751162662.ffe1934c280093c181c2b062a05e0908., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 , the old value is region=TestAcidGuarantees,,1731751162662.ffe1934c280093c181c2b062a05e0908., hostname=037c58a19ddb,37267,1731751077844, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffe1934c280093c181c2b062a05e0908, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:59:43,762 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731751162662.ffe1934c280093c181c2b062a05e0908., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffe1934c280093c181c2b062a05e0908, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:59:43,762 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731751162662.ffe1934c280093c181c2b062a05e0908., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 because the exception is null or not the one we care about 2024-11-16T09:59:43,823 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37877 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=109 2024-11-16T09:59:43,870 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=37267 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=110 2024-11-16T09:59:43,870 DEBUG [RS_FLUSH_OPERATIONS-regionserver/037c58a19ddb:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731751162662.ffe1934c280093c181c2b062a05e0908. 2024-11-16T09:59:43,870 DEBUG [RS_FLUSH_OPERATIONS-regionserver/037c58a19ddb:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731751162662.ffe1934c280093c181c2b062a05e0908. as already flushing 2024-11-16T09:59:43,870 DEBUG [RS_FLUSH_OPERATIONS-regionserver/037c58a19ddb:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731751162662.ffe1934c280093c181c2b062a05e0908. 2024-11-16T09:59:43,870 ERROR [RS_FLUSH_OPERATIONS-regionserver/037c58a19ddb:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] handler.RSProcedureHandler(58): pid=110 java.io.IOException: Unable to complete flush {ENCODED => ffe1934c280093c181c2b062a05e0908, NAME => 'TestAcidGuarantees,,1731751162662.ffe1934c280093c181c2b062a05e0908.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T09:59:43,871 DEBUG [RS_FLUSH_OPERATIONS-regionserver/037c58a19ddb:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=110 java.io.IOException: Unable to complete flush {ENCODED => ffe1934c280093c181c2b062a05e0908, NAME => 'TestAcidGuarantees,,1731751162662.ffe1934c280093c181c2b062a05e0908.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T09:59:43,871 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37877 {}] master.HMaster(4177): Remote procedure failed, pid=110 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ffe1934c280093c181c2b062a05e0908, NAME => 'TestAcidGuarantees,,1731751162662.ffe1934c280093c181c2b062a05e0908.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ffe1934c280093c181c2b062a05e0908, NAME => 'TestAcidGuarantees,,1731751162662.ffe1934c280093c181c2b062a05e0908.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T09:59:43,872 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffe1934c280093c181c2b062a05e0908, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T09:59:43,872 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] ipc.CallRunner(138): callId: 96 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43332 deadline: 1731751243862, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffe1934c280093c181c2b062a05e0908, server=037c58a19ddb,37267,1731751077844 2024-11-16T09:59:43,872 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731751162662.ffe1934c280093c181c2b062a05e0908., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 , the old value is region=TestAcidGuarantees,,1731751162662.ffe1934c280093c181c2b062a05e0908., hostname=037c58a19ddb,37267,1731751077844, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffe1934c280093c181c2b062a05e0908, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:59:43,872 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffe1934c280093c181c2b062a05e0908, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T09:59:43,872 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731751162662.ffe1934c280093c181c2b062a05e0908., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffe1934c280093c181c2b062a05e0908, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:59:43,872 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731751162662.ffe1934c280093c181c2b062a05e0908., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 because the exception is null or not the one we care about 2024-11-16T09:59:43,872 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] ipc.CallRunner(138): callId: 99 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43416 deadline: 1731751243863, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffe1934c280093c181c2b062a05e0908, server=037c58a19ddb,37267,1731751077844 2024-11-16T09:59:43,873 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731751162662.ffe1934c280093c181c2b062a05e0908., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 , the old value is region=TestAcidGuarantees,,1731751162662.ffe1934c280093c181c2b062a05e0908., hostname=037c58a19ddb,37267,1731751077844, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffe1934c280093c181c2b062a05e0908, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:59:43,873 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffe1934c280093c181c2b062a05e0908, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T09:59:43,873 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731751162662.ffe1934c280093c181c2b062a05e0908., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffe1934c280093c181c2b062a05e0908, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:59:43,873 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731751162662.ffe1934c280093c181c2b062a05e0908., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 because the exception is null or not the one we care about 2024-11-16T09:59:43,873 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] ipc.CallRunner(138): callId: 99 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43418 deadline: 1731751243863, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffe1934c280093c181c2b062a05e0908, server=037c58a19ddb,37267,1731751077844 2024-11-16T09:59:43,873 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffe1934c280093c181c2b062a05e0908, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T09:59:43,873 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] ipc.CallRunner(138): callId: 91 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43362 deadline: 1731751243863, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffe1934c280093c181c2b062a05e0908, server=037c58a19ddb,37267,1731751077844 2024-11-16T09:59:43,873 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731751162662.ffe1934c280093c181c2b062a05e0908., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 , the old value is region=TestAcidGuarantees,,1731751162662.ffe1934c280093c181c2b062a05e0908., hostname=037c58a19ddb,37267,1731751077844, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffe1934c280093c181c2b062a05e0908, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:59:43,873 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731751162662.ffe1934c280093c181c2b062a05e0908., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffe1934c280093c181c2b062a05e0908, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:59:43,873 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731751162662.ffe1934c280093c181c2b062a05e0908., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 because the exception is null or not the one we care about 2024-11-16T09:59:43,874 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731751162662.ffe1934c280093c181c2b062a05e0908., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 , the old value is region=TestAcidGuarantees,,1731751162662.ffe1934c280093c181c2b062a05e0908., hostname=037c58a19ddb,37267,1731751077844, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffe1934c280093c181c2b062a05e0908, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:59:43,874 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731751162662.ffe1934c280093c181c2b062a05e0908., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffe1934c280093c181c2b062a05e0908, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:59:43,874 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731751162662.ffe1934c280093c181c2b062a05e0908., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 because the exception is null or not the one we care about 2024-11-16T09:59:43,879 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffe1934c280093c181c2b062a05e0908, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T09:59:43,879 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] ipc.CallRunner(138): callId: 98 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43402 deadline: 1731751243873, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffe1934c280093c181c2b062a05e0908, server=037c58a19ddb,37267,1731751077844 2024-11-16T09:59:43,879 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731751162662.ffe1934c280093c181c2b062a05e0908., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 , the old value is region=TestAcidGuarantees,,1731751162662.ffe1934c280093c181c2b062a05e0908., hostname=037c58a19ddb,37267,1731751077844, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffe1934c280093c181c2b062a05e0908, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:59:43,879 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731751162662.ffe1934c280093c181c2b062a05e0908., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffe1934c280093c181c2b062a05e0908, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:59:43,879 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731751162662.ffe1934c280093c181c2b062a05e0908., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 because the exception is null or not the one we care about 2024-11-16T09:59:44,023 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=37267 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=110 2024-11-16T09:59:44,023 DEBUG [RS_FLUSH_OPERATIONS-regionserver/037c58a19ddb:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731751162662.ffe1934c280093c181c2b062a05e0908. 2024-11-16T09:59:44,024 DEBUG [RS_FLUSH_OPERATIONS-regionserver/037c58a19ddb:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731751162662.ffe1934c280093c181c2b062a05e0908. as already flushing 2024-11-16T09:59:44,024 DEBUG [RS_FLUSH_OPERATIONS-regionserver/037c58a19ddb:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731751162662.ffe1934c280093c181c2b062a05e0908. 2024-11-16T09:59:44,024 ERROR [RS_FLUSH_OPERATIONS-regionserver/037c58a19ddb:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] handler.RSProcedureHandler(58): pid=110 java.io.IOException: Unable to complete flush {ENCODED => ffe1934c280093c181c2b062a05e0908, NAME => 'TestAcidGuarantees,,1731751162662.ffe1934c280093c181c2b062a05e0908.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T09:59:44,024 DEBUG [RS_FLUSH_OPERATIONS-regionserver/037c58a19ddb:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=110 java.io.IOException: Unable to complete flush {ENCODED => ffe1934c280093c181c2b062a05e0908, NAME => 'TestAcidGuarantees,,1731751162662.ffe1934c280093c181c2b062a05e0908.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T09:59:44,025 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37877 {}] master.HMaster(4177): Remote procedure failed, pid=110 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ffe1934c280093c181c2b062a05e0908, NAME => 'TestAcidGuarantees,,1731751162662.ffe1934c280093c181c2b062a05e0908.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ffe1934c280093c181c2b062a05e0908, NAME => 'TestAcidGuarantees,,1731751162662.ffe1934c280093c181c2b062a05e0908.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T09:59:44,033 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37877 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=109 2024-11-16T09:59:44,092 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffe1934c280093c181c2b062a05e0908, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T09:59:44,092 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] ipc.CallRunner(138): callId: 97 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43332 deadline: 1731751244083, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffe1934c280093c181c2b062a05e0908, server=037c58a19ddb,37267,1731751077844 2024-11-16T09:59:44,092 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffe1934c280093c181c2b062a05e0908, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T09:59:44,092 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731751162662.ffe1934c280093c181c2b062a05e0908., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 , the old value is region=TestAcidGuarantees,,1731751162662.ffe1934c280093c181c2b062a05e0908., hostname=037c58a19ddb,37267,1731751077844, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffe1934c280093c181c2b062a05e0908, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:59:44,092 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731751162662.ffe1934c280093c181c2b062a05e0908., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffe1934c280093c181c2b062a05e0908, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:59:44,092 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] ipc.CallRunner(138): callId: 100 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43416 deadline: 1731751244083, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffe1934c280093c181c2b062a05e0908, server=037c58a19ddb,37267,1731751077844 2024-11-16T09:59:44,092 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731751162662.ffe1934c280093c181c2b062a05e0908., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 because the exception is null or not the one we care about 2024-11-16T09:59:44,093 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffe1934c280093c181c2b062a05e0908, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T09:59:44,093 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] ipc.CallRunner(138): callId: 100 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43418 deadline: 1731751244083, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffe1934c280093c181c2b062a05e0908, server=037c58a19ddb,37267,1731751077844 2024-11-16T09:59:44,093 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731751162662.ffe1934c280093c181c2b062a05e0908., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 , the old value is region=TestAcidGuarantees,,1731751162662.ffe1934c280093c181c2b062a05e0908., hostname=037c58a19ddb,37267,1731751077844, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffe1934c280093c181c2b062a05e0908, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:59:44,093 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731751162662.ffe1934c280093c181c2b062a05e0908., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffe1934c280093c181c2b062a05e0908, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:59:44,093 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731751162662.ffe1934c280093c181c2b062a05e0908., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 because the exception is null or not the one we care about 2024-11-16T09:59:44,093 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffe1934c280093c181c2b062a05e0908, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T09:59:44,093 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731751162662.ffe1934c280093c181c2b062a05e0908., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 , the old value is region=TestAcidGuarantees,,1731751162662.ffe1934c280093c181c2b062a05e0908., hostname=037c58a19ddb,37267,1731751077844, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffe1934c280093c181c2b062a05e0908, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:59:44,093 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] ipc.CallRunner(138): callId: 92 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43362 deadline: 1731751244083, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffe1934c280093c181c2b062a05e0908, server=037c58a19ddb,37267,1731751077844 2024-11-16T09:59:44,093 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731751162662.ffe1934c280093c181c2b062a05e0908., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffe1934c280093c181c2b062a05e0908, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:59:44,093 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731751162662.ffe1934c280093c181c2b062a05e0908., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 because the exception is null or not the one we care about 2024-11-16T09:59:44,093 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffe1934c280093c181c2b062a05e0908, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T09:59:44,093 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] ipc.CallRunner(138): callId: 99 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43402 deadline: 1731751244083, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffe1934c280093c181c2b062a05e0908, server=037c58a19ddb,37267,1731751077844 2024-11-16T09:59:44,093 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731751162662.ffe1934c280093c181c2b062a05e0908., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 , the old value is region=TestAcidGuarantees,,1731751162662.ffe1934c280093c181c2b062a05e0908., hostname=037c58a19ddb,37267,1731751077844, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffe1934c280093c181c2b062a05e0908, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:59:44,093 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731751162662.ffe1934c280093c181c2b062a05e0908., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffe1934c280093c181c2b062a05e0908, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:59:44,093 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731751162662.ffe1934c280093c181c2b062a05e0908., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 because the exception is null or not the one we care about 2024-11-16T09:59:44,093 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731751162662.ffe1934c280093c181c2b062a05e0908., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 , the old value is region=TestAcidGuarantees,,1731751162662.ffe1934c280093c181c2b062a05e0908., hostname=037c58a19ddb,37267,1731751077844, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffe1934c280093c181c2b062a05e0908, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:59:44,093 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731751162662.ffe1934c280093c181c2b062a05e0908., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffe1934c280093c181c2b062a05e0908, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:59:44,093 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731751162662.ffe1934c280093c181c2b062a05e0908., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 because the exception is null or not the one we care about 2024-11-16T09:59:44,117 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T09:59:44,121 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111656efabdd34124ec596e7ad480ca80c50_ffe1934c280093c181c2b062a05e0908 to hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111656efabdd34124ec596e7ad480ca80c50_ffe1934c280093c181c2b062a05e0908 2024-11-16T09:59:44,122 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/ffe1934c280093c181c2b062a05e0908/.tmp/A/e0361185f7ea491a8b4f44995bc8f102, store: [table=TestAcidGuarantees family=A region=ffe1934c280093c181c2b062a05e0908] 2024-11-16T09:59:44,122 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/ffe1934c280093c181c2b062a05e0908/.tmp/A/e0361185f7ea491a8b4f44995bc8f102 is 175, key is test_row_0/A:col10/1731751183703/Put/seqid=0 2024-11-16T09:59:44,126 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37591 is added to blk_1073742244_1420 (size=39949) 2024-11-16T09:59:44,176 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=37267 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=110 2024-11-16T09:59:44,177 DEBUG [RS_FLUSH_OPERATIONS-regionserver/037c58a19ddb:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731751162662.ffe1934c280093c181c2b062a05e0908. 2024-11-16T09:59:44,177 DEBUG [RS_FLUSH_OPERATIONS-regionserver/037c58a19ddb:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731751162662.ffe1934c280093c181c2b062a05e0908. as already flushing 2024-11-16T09:59:44,177 DEBUG [RS_FLUSH_OPERATIONS-regionserver/037c58a19ddb:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731751162662.ffe1934c280093c181c2b062a05e0908. 2024-11-16T09:59:44,177 ERROR [RS_FLUSH_OPERATIONS-regionserver/037c58a19ddb:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] handler.RSProcedureHandler(58): pid=110 java.io.IOException: Unable to complete flush {ENCODED => ffe1934c280093c181c2b062a05e0908, NAME => 'TestAcidGuarantees,,1731751162662.ffe1934c280093c181c2b062a05e0908.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T09:59:44,177 DEBUG [RS_FLUSH_OPERATIONS-regionserver/037c58a19ddb:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=110 java.io.IOException: Unable to complete flush {ENCODED => ffe1934c280093c181c2b062a05e0908, NAME => 'TestAcidGuarantees,,1731751162662.ffe1934c280093c181c2b062a05e0908.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T09:59:44,178 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37877 {}] master.HMaster(4177): Remote procedure failed, pid=110 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ffe1934c280093c181c2b062a05e0908, NAME => 'TestAcidGuarantees,,1731751162662.ffe1934c280093c181c2b062a05e0908.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ffe1934c280093c181c2b062a05e0908, NAME => 'TestAcidGuarantees,,1731751162662.ffe1934c280093c181c2b062a05e0908.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T09:59:44,330 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=37267 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=110 2024-11-16T09:59:44,330 DEBUG [RS_FLUSH_OPERATIONS-regionserver/037c58a19ddb:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731751162662.ffe1934c280093c181c2b062a05e0908. 2024-11-16T09:59:44,330 DEBUG [RS_FLUSH_OPERATIONS-regionserver/037c58a19ddb:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731751162662.ffe1934c280093c181c2b062a05e0908. as already flushing 2024-11-16T09:59:44,330 DEBUG [RS_FLUSH_OPERATIONS-regionserver/037c58a19ddb:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731751162662.ffe1934c280093c181c2b062a05e0908. 2024-11-16T09:59:44,330 ERROR [RS_FLUSH_OPERATIONS-regionserver/037c58a19ddb:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] handler.RSProcedureHandler(58): pid=110 java.io.IOException: Unable to complete flush {ENCODED => ffe1934c280093c181c2b062a05e0908, NAME => 'TestAcidGuarantees,,1731751162662.ffe1934c280093c181c2b062a05e0908.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T09:59:44,330 DEBUG [RS_FLUSH_OPERATIONS-regionserver/037c58a19ddb:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=110 java.io.IOException: Unable to complete flush {ENCODED => ffe1934c280093c181c2b062a05e0908, NAME => 'TestAcidGuarantees,,1731751162662.ffe1934c280093c181c2b062a05e0908.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T09:59:44,331 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37877 {}] master.HMaster(4177): Remote procedure failed, pid=110 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ffe1934c280093c181c2b062a05e0908, NAME => 'TestAcidGuarantees,,1731751162662.ffe1934c280093c181c2b062a05e0908.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ffe1934c280093c181c2b062a05e0908, NAME => 'TestAcidGuarantees,,1731751162662.ffe1934c280093c181c2b062a05e0908.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T09:59:44,343 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37877 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=109 2024-11-16T09:59:44,406 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffe1934c280093c181c2b062a05e0908, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T09:59:44,406 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] ipc.CallRunner(138): callId: 98 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43332 deadline: 1731751244403, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffe1934c280093c181c2b062a05e0908, server=037c58a19ddb,37267,1731751077844 2024-11-16T09:59:44,406 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffe1934c280093c181c2b062a05e0908, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T09:59:44,406 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] ipc.CallRunner(138): callId: 101 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43418 deadline: 1731751244403, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffe1934c280093c181c2b062a05e0908, server=037c58a19ddb,37267,1731751077844 2024-11-16T09:59:44,406 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731751162662.ffe1934c280093c181c2b062a05e0908., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 , the old value is region=TestAcidGuarantees,,1731751162662.ffe1934c280093c181c2b062a05e0908., hostname=037c58a19ddb,37267,1731751077844, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffe1934c280093c181c2b062a05e0908, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:59:44,406 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731751162662.ffe1934c280093c181c2b062a05e0908., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffe1934c280093c181c2b062a05e0908, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:59:44,406 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731751162662.ffe1934c280093c181c2b062a05e0908., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 because the exception is null or not the one we care about 2024-11-16T09:59:44,406 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffe1934c280093c181c2b062a05e0908, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T09:59:44,406 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] ipc.CallRunner(138): callId: 101 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43416 deadline: 1731751244403, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffe1934c280093c181c2b062a05e0908, server=037c58a19ddb,37267,1731751077844 2024-11-16T09:59:44,406 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731751162662.ffe1934c280093c181c2b062a05e0908., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 , the old value is region=TestAcidGuarantees,,1731751162662.ffe1934c280093c181c2b062a05e0908., hostname=037c58a19ddb,37267,1731751077844, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffe1934c280093c181c2b062a05e0908, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:59:44,406 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731751162662.ffe1934c280093c181c2b062a05e0908., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffe1934c280093c181c2b062a05e0908, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:59:44,406 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731751162662.ffe1934c280093c181c2b062a05e0908., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 because the exception is null or not the one we care about 2024-11-16T09:59:44,406 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffe1934c280093c181c2b062a05e0908, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T09:59:44,406 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731751162662.ffe1934c280093c181c2b062a05e0908., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 , the old value is region=TestAcidGuarantees,,1731751162662.ffe1934c280093c181c2b062a05e0908., hostname=037c58a19ddb,37267,1731751077844, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffe1934c280093c181c2b062a05e0908, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:59:44,406 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] ipc.CallRunner(138): callId: 100 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43402 deadline: 1731751244403, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffe1934c280093c181c2b062a05e0908, server=037c58a19ddb,37267,1731751077844 2024-11-16T09:59:44,407 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731751162662.ffe1934c280093c181c2b062a05e0908., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffe1934c280093c181c2b062a05e0908, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:59:44,407 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731751162662.ffe1934c280093c181c2b062a05e0908., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 because the exception is null or not the one we care about 2024-11-16T09:59:44,407 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffe1934c280093c181c2b062a05e0908, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T09:59:44,407 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731751162662.ffe1934c280093c181c2b062a05e0908., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 , the old value is region=TestAcidGuarantees,,1731751162662.ffe1934c280093c181c2b062a05e0908., hostname=037c58a19ddb,37267,1731751077844, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffe1934c280093c181c2b062a05e0908, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:59:44,407 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] ipc.CallRunner(138): callId: 93 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43362 deadline: 1731751244403, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffe1934c280093c181c2b062a05e0908, server=037c58a19ddb,37267,1731751077844 2024-11-16T09:59:44,407 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731751162662.ffe1934c280093c181c2b062a05e0908., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffe1934c280093c181c2b062a05e0908, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:59:44,407 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731751162662.ffe1934c280093c181c2b062a05e0908., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 because the exception is null or not the one we care about 2024-11-16T09:59:44,407 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731751162662.ffe1934c280093c181c2b062a05e0908., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 , the old value is region=TestAcidGuarantees,,1731751162662.ffe1934c280093c181c2b062a05e0908., hostname=037c58a19ddb,37267,1731751077844, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffe1934c280093c181c2b062a05e0908, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:59:44,407 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731751162662.ffe1934c280093c181c2b062a05e0908., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffe1934c280093c181c2b062a05e0908, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:59:44,407 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731751162662.ffe1934c280093c181c2b062a05e0908., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 because the exception is null or not the one we care about 2024-11-16T09:59:44,483 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=37267 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=110 2024-11-16T09:59:44,483 DEBUG [RS_FLUSH_OPERATIONS-regionserver/037c58a19ddb:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731751162662.ffe1934c280093c181c2b062a05e0908. 2024-11-16T09:59:44,484 DEBUG [RS_FLUSH_OPERATIONS-regionserver/037c58a19ddb:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731751162662.ffe1934c280093c181c2b062a05e0908. as already flushing 2024-11-16T09:59:44,484 DEBUG [RS_FLUSH_OPERATIONS-regionserver/037c58a19ddb:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731751162662.ffe1934c280093c181c2b062a05e0908. 2024-11-16T09:59:44,484 ERROR [RS_FLUSH_OPERATIONS-regionserver/037c58a19ddb:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] handler.RSProcedureHandler(58): pid=110 java.io.IOException: Unable to complete flush {ENCODED => ffe1934c280093c181c2b062a05e0908, NAME => 'TestAcidGuarantees,,1731751162662.ffe1934c280093c181c2b062a05e0908.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T09:59:44,484 DEBUG [RS_FLUSH_OPERATIONS-regionserver/037c58a19ddb:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=110 java.io.IOException: Unable to complete flush {ENCODED => ffe1934c280093c181c2b062a05e0908, NAME => 'TestAcidGuarantees,,1731751162662.ffe1934c280093c181c2b062a05e0908.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T09:59:44,484 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37877 {}] master.HMaster(4177): Remote procedure failed, pid=110 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ffe1934c280093c181c2b062a05e0908, NAME => 'TestAcidGuarantees,,1731751162662.ffe1934c280093c181c2b062a05e0908.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ffe1934c280093c181c2b062a05e0908, NAME => 'TestAcidGuarantees,,1731751162662.ffe1934c280093c181c2b062a05e0908.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T09:59:44,527 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=267, memsize=29.1 K, hasBloomFilter=true, into tmp file hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/ffe1934c280093c181c2b062a05e0908/.tmp/A/e0361185f7ea491a8b4f44995bc8f102 2024-11-16T09:59:44,534 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/ffe1934c280093c181c2b062a05e0908/.tmp/B/d208b33091da4873bc4b884835d2e72f is 50, key is test_row_0/B:col10/1731751183703/Put/seqid=0 2024-11-16T09:59:44,538 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37591 is added to blk_1073742245_1421 (size=12301) 2024-11-16T09:59:44,539 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=267 (bloomFilter=true), to=hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/ffe1934c280093c181c2b062a05e0908/.tmp/B/d208b33091da4873bc4b884835d2e72f 2024-11-16T09:59:44,546 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/ffe1934c280093c181c2b062a05e0908/.tmp/C/66cc8e420ade4f18ab524bd130018691 is 50, key is test_row_0/C:col10/1731751183703/Put/seqid=0 2024-11-16T09:59:44,549 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37591 is added to blk_1073742246_1422 (size=12301) 2024-11-16T09:59:44,550 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=267 (bloomFilter=true), to=hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/ffe1934c280093c181c2b062a05e0908/.tmp/C/66cc8e420ade4f18ab524bd130018691 2024-11-16T09:59:44,553 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/ffe1934c280093c181c2b062a05e0908/.tmp/A/e0361185f7ea491a8b4f44995bc8f102 as hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/ffe1934c280093c181c2b062a05e0908/A/e0361185f7ea491a8b4f44995bc8f102 2024-11-16T09:59:44,557 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/ffe1934c280093c181c2b062a05e0908/A/e0361185f7ea491a8b4f44995bc8f102, entries=200, sequenceid=267, filesize=39.0 K 2024-11-16T09:59:44,557 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/ffe1934c280093c181c2b062a05e0908/.tmp/B/d208b33091da4873bc4b884835d2e72f as hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/ffe1934c280093c181c2b062a05e0908/B/d208b33091da4873bc4b884835d2e72f 2024-11-16T09:59:44,561 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/ffe1934c280093c181c2b062a05e0908/B/d208b33091da4873bc4b884835d2e72f, entries=150, sequenceid=267, filesize=12.0 K 2024-11-16T09:59:44,561 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/ffe1934c280093c181c2b062a05e0908/.tmp/C/66cc8e420ade4f18ab524bd130018691 as hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/ffe1934c280093c181c2b062a05e0908/C/66cc8e420ade4f18ab524bd130018691 2024-11-16T09:59:44,565 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/ffe1934c280093c181c2b062a05e0908/C/66cc8e420ade4f18ab524bd130018691, entries=150, sequenceid=267, filesize=12.0 K 2024-11-16T09:59:44,566 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~87.22 KB/89310, heapSize ~229.22 KB/234720, currentSize=114.05 KB/116790 for ffe1934c280093c181c2b062a05e0908 in 862ms, sequenceid=267, compaction requested=true 2024-11-16T09:59:44,566 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for ffe1934c280093c181c2b062a05e0908: 2024-11-16T09:59:44,566 DEBUG [RS:0;037c58a19ddb:37267-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-16T09:59:44,566 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store ffe1934c280093c181c2b062a05e0908:A, priority=-2147483648, current under compaction store size is 1 2024-11-16T09:59:44,566 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-16T09:59:44,566 DEBUG [RS:0;037c58a19ddb:37267-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-16T09:59:44,567 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store ffe1934c280093c181c2b062a05e0908:B, priority=-2147483648, current under compaction store size is 2 2024-11-16T09:59:44,567 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-16T09:59:44,567 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store ffe1934c280093c181c2b062a05e0908:C, priority=-2147483648, current under compaction store size is 3 2024-11-16T09:59:44,567 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-16T09:59:44,568 DEBUG [RS:0;037c58a19ddb:37267-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 102705 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-16T09:59:44,568 DEBUG [RS:0;037c58a19ddb:37267-shortCompactions-0 {}] regionserver.HStore(1541): ffe1934c280093c181c2b062a05e0908/A is initiating minor compaction (all files) 2024-11-16T09:59:44,568 DEBUG [RS:0;037c58a19ddb:37267-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37149 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-16T09:59:44,568 INFO [RS:0;037c58a19ddb:37267-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of ffe1934c280093c181c2b062a05e0908/A in TestAcidGuarantees,,1731751162662.ffe1934c280093c181c2b062a05e0908. 2024-11-16T09:59:44,568 DEBUG [RS:0;037c58a19ddb:37267-longCompactions-0 {}] regionserver.HStore(1541): ffe1934c280093c181c2b062a05e0908/B is initiating minor compaction (all files) 2024-11-16T09:59:44,569 INFO [RS:0;037c58a19ddb:37267-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of ffe1934c280093c181c2b062a05e0908/B in TestAcidGuarantees,,1731751162662.ffe1934c280093c181c2b062a05e0908. 2024-11-16T09:59:44,569 INFO [RS:0;037c58a19ddb:37267-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/ffe1934c280093c181c2b062a05e0908/A/2bbc77228e7446d4826467c6853a1981, hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/ffe1934c280093c181c2b062a05e0908/A/dac023d72c9a467cb301eb07f30695ca, hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/ffe1934c280093c181c2b062a05e0908/A/e0361185f7ea491a8b4f44995bc8f102] into tmpdir=hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/ffe1934c280093c181c2b062a05e0908/.tmp, totalSize=100.3 K 2024-11-16T09:59:44,569 INFO [RS:0;037c58a19ddb:37267-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/ffe1934c280093c181c2b062a05e0908/B/04b0e51d058b49c7ac39bd0526239b72, hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/ffe1934c280093c181c2b062a05e0908/B/dc55e610407e49158964b1ae68c11f86, hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/ffe1934c280093c181c2b062a05e0908/B/d208b33091da4873bc4b884835d2e72f] into tmpdir=hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/ffe1934c280093c181c2b062a05e0908/.tmp, totalSize=36.3 K 2024-11-16T09:59:44,569 INFO [RS:0;037c58a19ddb:37267-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731751162662.ffe1934c280093c181c2b062a05e0908. 2024-11-16T09:59:44,569 DEBUG [RS:0;037c58a19ddb:37267-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731751162662.ffe1934c280093c181c2b062a05e0908. files: [hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/ffe1934c280093c181c2b062a05e0908/A/2bbc77228e7446d4826467c6853a1981, hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/ffe1934c280093c181c2b062a05e0908/A/dac023d72c9a467cb301eb07f30695ca, hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/ffe1934c280093c181c2b062a05e0908/A/e0361185f7ea491a8b4f44995bc8f102] 2024-11-16T09:59:44,569 DEBUG [RS:0;037c58a19ddb:37267-longCompactions-0 {}] compactions.Compactor(225): Compacting 04b0e51d058b49c7ac39bd0526239b72, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=227, earliestPutTs=1731751179681 2024-11-16T09:59:44,569 DEBUG [RS:0;037c58a19ddb:37267-shortCompactions-0 {}] compactions.Compactor(225): Compacting 2bbc77228e7446d4826467c6853a1981, keycount=150, bloomtype=ROW, size=30.9 K, encoding=NONE, compression=NONE, seqNum=227, earliestPutTs=1731751179681 2024-11-16T09:59:44,569 DEBUG [RS:0;037c58a19ddb:37267-longCompactions-0 {}] compactions.Compactor(225): Compacting dc55e610407e49158964b1ae68c11f86, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=248, earliestPutTs=1731751180874 2024-11-16T09:59:44,569 DEBUG [RS:0;037c58a19ddb:37267-shortCompactions-0 {}] compactions.Compactor(225): Compacting dac023d72c9a467cb301eb07f30695ca, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=248, earliestPutTs=1731751180874 2024-11-16T09:59:44,570 DEBUG [RS:0;037c58a19ddb:37267-longCompactions-0 {}] compactions.Compactor(225): Compacting d208b33091da4873bc4b884835d2e72f, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=267, earliestPutTs=1731751183067 2024-11-16T09:59:44,570 DEBUG [RS:0;037c58a19ddb:37267-shortCompactions-0 {}] compactions.Compactor(225): Compacting e0361185f7ea491a8b4f44995bc8f102, keycount=200, bloomtype=ROW, size=39.0 K, encoding=NONE, compression=NONE, seqNum=267, earliestPutTs=1731751183067 2024-11-16T09:59:44,576 INFO [RS:0;037c58a19ddb:37267-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=ffe1934c280093c181c2b062a05e0908] 2024-11-16T09:59:44,578 INFO [RS:0;037c58a19ddb:37267-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): ffe1934c280093c181c2b062a05e0908#B#compaction#356 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-16T09:59:44,578 DEBUG [RS:0;037c58a19ddb:37267-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e2024111654b58f4940274184ba777a5ff26b9bb2_ffe1934c280093c181c2b062a05e0908 store=[table=TestAcidGuarantees family=A region=ffe1934c280093c181c2b062a05e0908] 2024-11-16T09:59:44,578 DEBUG [RS:0;037c58a19ddb:37267-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/ffe1934c280093c181c2b062a05e0908/.tmp/B/2bd7c66293374d64944f326ac038a69b is 50, key is test_row_0/B:col10/1731751183703/Put/seqid=0 2024-11-16T09:59:44,579 DEBUG [RS:0;037c58a19ddb:37267-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e2024111654b58f4940274184ba777a5ff26b9bb2_ffe1934c280093c181c2b062a05e0908, store=[table=TestAcidGuarantees family=A region=ffe1934c280093c181c2b062a05e0908] 2024-11-16T09:59:44,579 DEBUG [RS:0;037c58a19ddb:37267-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111654b58f4940274184ba777a5ff26b9bb2_ffe1934c280093c181c2b062a05e0908 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=ffe1934c280093c181c2b062a05e0908] 2024-11-16T09:59:44,584 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37591 is added to blk_1073742247_1423 (size=12949) 2024-11-16T09:59:44,585 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37591 is added to blk_1073742248_1424 (size=4469) 2024-11-16T09:59:44,636 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=37267 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=110 2024-11-16T09:59:44,637 DEBUG [RS_FLUSH_OPERATIONS-regionserver/037c58a19ddb:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731751162662.ffe1934c280093c181c2b062a05e0908. 2024-11-16T09:59:44,637 INFO [RS_FLUSH_OPERATIONS-regionserver/037c58a19ddb:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.HRegion(2902): Flushing ffe1934c280093c181c2b062a05e0908 3/3 column families, dataSize=114.05 KB heapSize=299.58 KB 2024-11-16T09:59:44,643 DEBUG [RS_FLUSH_OPERATIONS-regionserver/037c58a19ddb:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241116f2e8ccce4d6b45ff8661f0b31a5e0b62_ffe1934c280093c181c2b062a05e0908 is 50, key is test_row_0/A:col10/1731751183750/Put/seqid=0 2024-11-16T09:59:44,646 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37591 is added to blk_1073742249_1425 (size=12454) 2024-11-16T09:59:44,852 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37877 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=109 2024-11-16T09:59:44,917 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] regionserver.HRegion(8855): Flush requested on ffe1934c280093c181c2b062a05e0908 2024-11-16T09:59:44,918 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731751162662.ffe1934c280093c181c2b062a05e0908. as already flushing 2024-11-16T09:59:44,950 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffe1934c280093c181c2b062a05e0908, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T09:59:44,950 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] ipc.CallRunner(138): callId: 104 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43416 deadline: 1731751244933, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffe1934c280093c181c2b062a05e0908, server=037c58a19ddb,37267,1731751077844 2024-11-16T09:59:44,950 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731751162662.ffe1934c280093c181c2b062a05e0908., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 , the old value is region=TestAcidGuarantees,,1731751162662.ffe1934c280093c181c2b062a05e0908., hostname=037c58a19ddb,37267,1731751077844, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffe1934c280093c181c2b062a05e0908, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:59:44,950 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffe1934c280093c181c2b062a05e0908, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T09:59:44,950 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731751162662.ffe1934c280093c181c2b062a05e0908., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffe1934c280093c181c2b062a05e0908, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:59:44,950 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731751162662.ffe1934c280093c181c2b062a05e0908., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 because the exception is null or not the one we care about 2024-11-16T09:59:44,950 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] ipc.CallRunner(138): callId: 96 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43362 deadline: 1731751244933, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffe1934c280093c181c2b062a05e0908, server=037c58a19ddb,37267,1731751077844 2024-11-16T09:59:44,951 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffe1934c280093c181c2b062a05e0908, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T09:59:44,951 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731751162662.ffe1934c280093c181c2b062a05e0908., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 , the old value is region=TestAcidGuarantees,,1731751162662.ffe1934c280093c181c2b062a05e0908., hostname=037c58a19ddb,37267,1731751077844, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffe1934c280093c181c2b062a05e0908, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:59:44,951 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] ipc.CallRunner(138): callId: 102 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43332 deadline: 1731751244935, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffe1934c280093c181c2b062a05e0908, server=037c58a19ddb,37267,1731751077844 2024-11-16T09:59:44,951 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731751162662.ffe1934c280093c181c2b062a05e0908., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffe1934c280093c181c2b062a05e0908, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:59:44,951 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731751162662.ffe1934c280093c181c2b062a05e0908., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 because the exception is null or not the one we care about 2024-11-16T09:59:44,951 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731751162662.ffe1934c280093c181c2b062a05e0908., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 , the old value is region=TestAcidGuarantees,,1731751162662.ffe1934c280093c181c2b062a05e0908., hostname=037c58a19ddb,37267,1731751077844, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffe1934c280093c181c2b062a05e0908, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:59:44,951 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffe1934c280093c181c2b062a05e0908, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T09:59:44,951 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731751162662.ffe1934c280093c181c2b062a05e0908., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffe1934c280093c181c2b062a05e0908, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:59:44,951 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731751162662.ffe1934c280093c181c2b062a05e0908., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 because the exception is null or not the one we care about 2024-11-16T09:59:44,951 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] ipc.CallRunner(138): callId: 105 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43418 deadline: 1731751244936, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffe1934c280093c181c2b062a05e0908, server=037c58a19ddb,37267,1731751077844 2024-11-16T09:59:44,951 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731751162662.ffe1934c280093c181c2b062a05e0908., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 , the old value is region=TestAcidGuarantees,,1731751162662.ffe1934c280093c181c2b062a05e0908., hostname=037c58a19ddb,37267,1731751077844, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffe1934c280093c181c2b062a05e0908, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:59:44,951 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731751162662.ffe1934c280093c181c2b062a05e0908., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffe1934c280093c181c2b062a05e0908, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:59:44,951 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731751162662.ffe1934c280093c181c2b062a05e0908., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 because the exception is null or not the one we care about 2024-11-16T09:59:44,958 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffe1934c280093c181c2b062a05e0908, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T09:59:44,958 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] ipc.CallRunner(138): callId: 104 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43402 deadline: 1731751244950, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffe1934c280093c181c2b062a05e0908, server=037c58a19ddb,37267,1731751077844 2024-11-16T09:59:44,958 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731751162662.ffe1934c280093c181c2b062a05e0908., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 , the old value is region=TestAcidGuarantees,,1731751162662.ffe1934c280093c181c2b062a05e0908., hostname=037c58a19ddb,37267,1731751077844, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffe1934c280093c181c2b062a05e0908, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:59:44,959 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731751162662.ffe1934c280093c181c2b062a05e0908., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffe1934c280093c181c2b062a05e0908, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:59:44,959 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731751162662.ffe1934c280093c181c2b062a05e0908., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 because the exception is null or not the one we care about 2024-11-16T09:59:44,986 INFO [RS:0;037c58a19ddb:37267-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): ffe1934c280093c181c2b062a05e0908#A#compaction#357 average throughput is 0.06 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-16T09:59:44,986 DEBUG [RS:0;037c58a19ddb:37267-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/ffe1934c280093c181c2b062a05e0908/.tmp/A/8f7d13fd125a4e48ba6db638d37dd1d3 is 175, key is test_row_0/A:col10/1731751183703/Put/seqid=0 2024-11-16T09:59:44,991 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37591 is added to blk_1073742250_1426 (size=31903) 2024-11-16T09:59:44,991 DEBUG [RS:0;037c58a19ddb:37267-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/ffe1934c280093c181c2b062a05e0908/.tmp/B/2bd7c66293374d64944f326ac038a69b as hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/ffe1934c280093c181c2b062a05e0908/B/2bd7c66293374d64944f326ac038a69b 2024-11-16T09:59:44,995 INFO [RS:0;037c58a19ddb:37267-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in ffe1934c280093c181c2b062a05e0908/B of ffe1934c280093c181c2b062a05e0908 into 2bd7c66293374d64944f326ac038a69b(size=12.6 K), total size for store is 12.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-16T09:59:44,996 DEBUG [RS:0;037c58a19ddb:37267-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for ffe1934c280093c181c2b062a05e0908: 2024-11-16T09:59:44,996 INFO [RS:0;037c58a19ddb:37267-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731751162662.ffe1934c280093c181c2b062a05e0908., storeName=ffe1934c280093c181c2b062a05e0908/B, priority=13, startTime=1731751184566; duration=0sec 2024-11-16T09:59:44,996 DEBUG [RS:0;037c58a19ddb:37267-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-16T09:59:44,996 DEBUG [RS:0;037c58a19ddb:37267-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: ffe1934c280093c181c2b062a05e0908:B 2024-11-16T09:59:44,996 DEBUG [RS:0;037c58a19ddb:37267-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-16T09:59:44,997 DEBUG [RS:0;037c58a19ddb:37267-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37149 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-16T09:59:44,997 DEBUG [RS:0;037c58a19ddb:37267-longCompactions-0 {}] regionserver.HStore(1541): ffe1934c280093c181c2b062a05e0908/C is initiating minor compaction (all files) 2024-11-16T09:59:44,997 INFO [RS:0;037c58a19ddb:37267-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of ffe1934c280093c181c2b062a05e0908/C in TestAcidGuarantees,,1731751162662.ffe1934c280093c181c2b062a05e0908. 2024-11-16T09:59:44,997 INFO [RS:0;037c58a19ddb:37267-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/ffe1934c280093c181c2b062a05e0908/C/c498d0fe177440029170dc63c55c9c94, hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/ffe1934c280093c181c2b062a05e0908/C/0f4fae61ac1e4e00bebc2ddf2341d072, hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/ffe1934c280093c181c2b062a05e0908/C/66cc8e420ade4f18ab524bd130018691] into tmpdir=hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/ffe1934c280093c181c2b062a05e0908/.tmp, totalSize=36.3 K 2024-11-16T09:59:44,997 DEBUG [RS:0;037c58a19ddb:37267-longCompactions-0 {}] compactions.Compactor(225): Compacting c498d0fe177440029170dc63c55c9c94, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=227, earliestPutTs=1731751179681 2024-11-16T09:59:44,997 DEBUG [RS:0;037c58a19ddb:37267-longCompactions-0 {}] compactions.Compactor(225): Compacting 0f4fae61ac1e4e00bebc2ddf2341d072, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=248, earliestPutTs=1731751180874 2024-11-16T09:59:44,998 DEBUG [RS:0;037c58a19ddb:37267-longCompactions-0 {}] compactions.Compactor(225): Compacting 66cc8e420ade4f18ab524bd130018691, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=267, earliestPutTs=1731751183067 2024-11-16T09:59:45,003 INFO [RS:0;037c58a19ddb:37267-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): ffe1934c280093c181c2b062a05e0908#C#compaction#359 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-16T09:59:45,004 DEBUG [RS:0;037c58a19ddb:37267-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/ffe1934c280093c181c2b062a05e0908/.tmp/C/41feadaaa74748549c2f8cde91937b45 is 50, key is test_row_0/C:col10/1731751183703/Put/seqid=0 2024-11-16T09:59:45,007 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37591 is added to blk_1073742251_1427 (size=12949) 2024-11-16T09:59:45,047 DEBUG [RS_FLUSH_OPERATIONS-regionserver/037c58a19ddb:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T09:59:45,051 INFO [RS_FLUSH_OPERATIONS-regionserver/037c58a19ddb:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241116f2e8ccce4d6b45ff8661f0b31a5e0b62_ffe1934c280093c181c2b062a05e0908 to hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241116f2e8ccce4d6b45ff8661f0b31a5e0b62_ffe1934c280093c181c2b062a05e0908 2024-11-16T09:59:45,051 DEBUG [RS_FLUSH_OPERATIONS-regionserver/037c58a19ddb:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/ffe1934c280093c181c2b062a05e0908/.tmp/A/fedbe5ffd79d430bbbbbeb61c1fe203e, store: [table=TestAcidGuarantees family=A region=ffe1934c280093c181c2b062a05e0908] 2024-11-16T09:59:45,052 DEBUG [RS_FLUSH_OPERATIONS-regionserver/037c58a19ddb:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/ffe1934c280093c181c2b062a05e0908/.tmp/A/fedbe5ffd79d430bbbbbeb61c1fe203e is 175, key is test_row_0/A:col10/1731751183750/Put/seqid=0 2024-11-16T09:59:45,056 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37591 is added to blk_1073742252_1428 (size=31255) 2024-11-16T09:59:45,056 INFO [RS_FLUSH_OPERATIONS-regionserver/037c58a19ddb:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=287, memsize=38.0 K, hasBloomFilter=true, into tmp file hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/ffe1934c280093c181c2b062a05e0908/.tmp/A/fedbe5ffd79d430bbbbbeb61c1fe203e 2024-11-16T09:59:45,062 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffe1934c280093c181c2b062a05e0908, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T09:59:45,062 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] ipc.CallRunner(138): callId: 105 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43416 deadline: 1731751245052, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffe1934c280093c181c2b062a05e0908, server=037c58a19ddb,37267,1731751077844 2024-11-16T09:59:45,063 DEBUG [RS_FLUSH_OPERATIONS-regionserver/037c58a19ddb:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/ffe1934c280093c181c2b062a05e0908/.tmp/B/a6e336fc389343dab5c447e05135b43f is 50, key is test_row_0/B:col10/1731751183750/Put/seqid=0 2024-11-16T09:59:45,063 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731751162662.ffe1934c280093c181c2b062a05e0908., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 , the old value is region=TestAcidGuarantees,,1731751162662.ffe1934c280093c181c2b062a05e0908., hostname=037c58a19ddb,37267,1731751077844, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffe1934c280093c181c2b062a05e0908, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:59:45,063 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731751162662.ffe1934c280093c181c2b062a05e0908., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffe1934c280093c181c2b062a05e0908, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:59:45,063 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffe1934c280093c181c2b062a05e0908, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T09:59:45,063 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731751162662.ffe1934c280093c181c2b062a05e0908., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 because the exception is null or not the one we care about 2024-11-16T09:59:45,063 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] ipc.CallRunner(138): callId: 97 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43362 deadline: 1731751245052, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffe1934c280093c181c2b062a05e0908, server=037c58a19ddb,37267,1731751077844 2024-11-16T09:59:45,063 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffe1934c280093c181c2b062a05e0908, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T09:59:45,063 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] ipc.CallRunner(138): callId: 103 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43332 deadline: 1731751245053, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffe1934c280093c181c2b062a05e0908, server=037c58a19ddb,37267,1731751077844 2024-11-16T09:59:45,064 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731751162662.ffe1934c280093c181c2b062a05e0908., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 , the old value is region=TestAcidGuarantees,,1731751162662.ffe1934c280093c181c2b062a05e0908., hostname=037c58a19ddb,37267,1731751077844, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffe1934c280093c181c2b062a05e0908, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:59:45,064 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731751162662.ffe1934c280093c181c2b062a05e0908., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffe1934c280093c181c2b062a05e0908, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:59:45,064 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731751162662.ffe1934c280093c181c2b062a05e0908., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 , the old value is region=TestAcidGuarantees,,1731751162662.ffe1934c280093c181c2b062a05e0908., hostname=037c58a19ddb,37267,1731751077844, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffe1934c280093c181c2b062a05e0908, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:59:45,064 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731751162662.ffe1934c280093c181c2b062a05e0908., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 because the exception is null or not the one we care about 2024-11-16T09:59:45,064 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731751162662.ffe1934c280093c181c2b062a05e0908., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffe1934c280093c181c2b062a05e0908, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:59:45,064 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731751162662.ffe1934c280093c181c2b062a05e0908., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 because the exception is null or not the one we care about 2024-11-16T09:59:45,067 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37591 is added to blk_1073742253_1429 (size=12301) 2024-11-16T09:59:45,072 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffe1934c280093c181c2b062a05e0908, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T09:59:45,072 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] ipc.CallRunner(138): callId: 106 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43418 deadline: 1731751245063, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffe1934c280093c181c2b062a05e0908, server=037c58a19ddb,37267,1731751077844 2024-11-16T09:59:45,073 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731751162662.ffe1934c280093c181c2b062a05e0908., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 , the old value is region=TestAcidGuarantees,,1731751162662.ffe1934c280093c181c2b062a05e0908., hostname=037c58a19ddb,37267,1731751077844, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffe1934c280093c181c2b062a05e0908, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:59:45,073 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffe1934c280093c181c2b062a05e0908, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T09:59:45,073 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731751162662.ffe1934c280093c181c2b062a05e0908., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffe1934c280093c181c2b062a05e0908, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:59:45,073 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731751162662.ffe1934c280093c181c2b062a05e0908., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 because the exception is null or not the one we care about 2024-11-16T09:59:45,073 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] ipc.CallRunner(138): callId: 105 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43402 deadline: 1731751245064, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffe1934c280093c181c2b062a05e0908, server=037c58a19ddb,37267,1731751077844 2024-11-16T09:59:45,073 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731751162662.ffe1934c280093c181c2b062a05e0908., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 , the old value is region=TestAcidGuarantees,,1731751162662.ffe1934c280093c181c2b062a05e0908., hostname=037c58a19ddb,37267,1731751077844, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffe1934c280093c181c2b062a05e0908, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:59:45,073 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731751162662.ffe1934c280093c181c2b062a05e0908., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffe1934c280093c181c2b062a05e0908, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:59:45,073 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731751162662.ffe1934c280093c181c2b062a05e0908., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 because the exception is null or not the one we care about 2024-11-16T09:59:45,278 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffe1934c280093c181c2b062a05e0908, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T09:59:45,278 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] ipc.CallRunner(138): callId: 106 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43416 deadline: 1731751245273, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffe1934c280093c181c2b062a05e0908, server=037c58a19ddb,37267,1731751077844 2024-11-16T09:59:45,278 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffe1934c280093c181c2b062a05e0908, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T09:59:45,278 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731751162662.ffe1934c280093c181c2b062a05e0908., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 , the old value is region=TestAcidGuarantees,,1731751162662.ffe1934c280093c181c2b062a05e0908., hostname=037c58a19ddb,37267,1731751077844, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffe1934c280093c181c2b062a05e0908, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:59:45,278 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731751162662.ffe1934c280093c181c2b062a05e0908., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffe1934c280093c181c2b062a05e0908, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:59:45,278 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] ipc.CallRunner(138): callId: 98 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43362 deadline: 1731751245273, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffe1934c280093c181c2b062a05e0908, server=037c58a19ddb,37267,1731751077844 2024-11-16T09:59:45,278 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731751162662.ffe1934c280093c181c2b062a05e0908., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 because the exception is null or not the one we care about 2024-11-16T09:59:45,278 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731751162662.ffe1934c280093c181c2b062a05e0908., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 , the old value is region=TestAcidGuarantees,,1731751162662.ffe1934c280093c181c2b062a05e0908., hostname=037c58a19ddb,37267,1731751077844, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffe1934c280093c181c2b062a05e0908, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:59:45,278 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffe1934c280093c181c2b062a05e0908, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T09:59:45,278 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731751162662.ffe1934c280093c181c2b062a05e0908., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffe1934c280093c181c2b062a05e0908, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:59:45,279 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731751162662.ffe1934c280093c181c2b062a05e0908., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 because the exception is null or not the one we care about 2024-11-16T09:59:45,279 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] ipc.CallRunner(138): callId: 104 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43332 deadline: 1731751245273, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffe1934c280093c181c2b062a05e0908, server=037c58a19ddb,37267,1731751077844 2024-11-16T09:59:45,279 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731751162662.ffe1934c280093c181c2b062a05e0908., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 , the old value is region=TestAcidGuarantees,,1731751162662.ffe1934c280093c181c2b062a05e0908., hostname=037c58a19ddb,37267,1731751077844, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffe1934c280093c181c2b062a05e0908, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:59:45,279 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731751162662.ffe1934c280093c181c2b062a05e0908., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffe1934c280093c181c2b062a05e0908, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:59:45,279 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731751162662.ffe1934c280093c181c2b062a05e0908., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 because the exception is null or not the one we care about 2024-11-16T09:59:45,289 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffe1934c280093c181c2b062a05e0908, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T09:59:45,290 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] ipc.CallRunner(138): callId: 107 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43418 deadline: 1731751245283, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffe1934c280093c181c2b062a05e0908, server=037c58a19ddb,37267,1731751077844 2024-11-16T09:59:45,290 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731751162662.ffe1934c280093c181c2b062a05e0908., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 , the old value is region=TestAcidGuarantees,,1731751162662.ffe1934c280093c181c2b062a05e0908., hostname=037c58a19ddb,37267,1731751077844, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffe1934c280093c181c2b062a05e0908, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:59:45,290 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731751162662.ffe1934c280093c181c2b062a05e0908., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffe1934c280093c181c2b062a05e0908, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:59:45,290 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731751162662.ffe1934c280093c181c2b062a05e0908., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 because the exception is null or not the one we care about 2024-11-16T09:59:45,290 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffe1934c280093c181c2b062a05e0908, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T09:59:45,290 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] ipc.CallRunner(138): callId: 106 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43402 deadline: 1731751245283, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffe1934c280093c181c2b062a05e0908, server=037c58a19ddb,37267,1731751077844 2024-11-16T09:59:45,290 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731751162662.ffe1934c280093c181c2b062a05e0908., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 , the old value is region=TestAcidGuarantees,,1731751162662.ffe1934c280093c181c2b062a05e0908., hostname=037c58a19ddb,37267,1731751077844, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffe1934c280093c181c2b062a05e0908, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:59:45,290 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731751162662.ffe1934c280093c181c2b062a05e0908., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffe1934c280093c181c2b062a05e0908, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:59:45,290 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731751162662.ffe1934c280093c181c2b062a05e0908., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 because the exception is null or not the one we care about 2024-11-16T09:59:45,396 DEBUG [RS:0;037c58a19ddb:37267-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/ffe1934c280093c181c2b062a05e0908/.tmp/A/8f7d13fd125a4e48ba6db638d37dd1d3 as hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/ffe1934c280093c181c2b062a05e0908/A/8f7d13fd125a4e48ba6db638d37dd1d3 2024-11-16T09:59:45,401 INFO [RS:0;037c58a19ddb:37267-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in ffe1934c280093c181c2b062a05e0908/A of ffe1934c280093c181c2b062a05e0908 into 8f7d13fd125a4e48ba6db638d37dd1d3(size=31.2 K), total size for store is 31.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-16T09:59:45,401 DEBUG [RS:0;037c58a19ddb:37267-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for ffe1934c280093c181c2b062a05e0908: 2024-11-16T09:59:45,401 INFO [RS:0;037c58a19ddb:37267-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731751162662.ffe1934c280093c181c2b062a05e0908., storeName=ffe1934c280093c181c2b062a05e0908/A, priority=13, startTime=1731751184566; duration=0sec 2024-11-16T09:59:45,401 DEBUG [RS:0;037c58a19ddb:37267-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-16T09:59:45,401 DEBUG [RS:0;037c58a19ddb:37267-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: ffe1934c280093c181c2b062a05e0908:A 2024-11-16T09:59:45,411 DEBUG [RS:0;037c58a19ddb:37267-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/ffe1934c280093c181c2b062a05e0908/.tmp/C/41feadaaa74748549c2f8cde91937b45 as hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/ffe1934c280093c181c2b062a05e0908/C/41feadaaa74748549c2f8cde91937b45 2024-11-16T09:59:45,415 INFO [RS:0;037c58a19ddb:37267-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in ffe1934c280093c181c2b062a05e0908/C of ffe1934c280093c181c2b062a05e0908 into 41feadaaa74748549c2f8cde91937b45(size=12.6 K), total size for store is 12.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-16T09:59:45,415 DEBUG [RS:0;037c58a19ddb:37267-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for ffe1934c280093c181c2b062a05e0908: 2024-11-16T09:59:45,415 INFO [RS:0;037c58a19ddb:37267-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731751162662.ffe1934c280093c181c2b062a05e0908., storeName=ffe1934c280093c181c2b062a05e0908/C, priority=13, startTime=1731751184567; duration=0sec 2024-11-16T09:59:45,416 DEBUG [RS:0;037c58a19ddb:37267-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-16T09:59:45,416 DEBUG [RS:0;037c58a19ddb:37267-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: ffe1934c280093c181c2b062a05e0908:C 2024-11-16T09:59:45,468 INFO [RS_FLUSH_OPERATIONS-regionserver/037c58a19ddb:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=287 (bloomFilter=true), to=hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/ffe1934c280093c181c2b062a05e0908/.tmp/B/a6e336fc389343dab5c447e05135b43f 2024-11-16T09:59:45,474 DEBUG [RS_FLUSH_OPERATIONS-regionserver/037c58a19ddb:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/ffe1934c280093c181c2b062a05e0908/.tmp/C/92d4b326ce914ac591b817dc8a4ea75f is 50, key is test_row_0/C:col10/1731751183750/Put/seqid=0 2024-11-16T09:59:45,478 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37591 is added to blk_1073742254_1430 (size=12301) 2024-11-16T09:59:45,589 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffe1934c280093c181c2b062a05e0908, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T09:59:45,589 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] ipc.CallRunner(138): callId: 107 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43416 deadline: 1731751245582, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffe1934c280093c181c2b062a05e0908, server=037c58a19ddb,37267,1731751077844 2024-11-16T09:59:45,590 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffe1934c280093c181c2b062a05e0908, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T09:59:45,590 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731751162662.ffe1934c280093c181c2b062a05e0908., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 , the old value is region=TestAcidGuarantees,,1731751162662.ffe1934c280093c181c2b062a05e0908., hostname=037c58a19ddb,37267,1731751077844, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffe1934c280093c181c2b062a05e0908, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:59:45,590 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] ipc.CallRunner(138): callId: 99 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43362 deadline: 1731751245582, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffe1934c280093c181c2b062a05e0908, server=037c58a19ddb,37267,1731751077844 2024-11-16T09:59:45,590 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731751162662.ffe1934c280093c181c2b062a05e0908., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffe1934c280093c181c2b062a05e0908, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:59:45,590 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731751162662.ffe1934c280093c181c2b062a05e0908., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 because the exception is null or not the one we care about 2024-11-16T09:59:45,590 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffe1934c280093c181c2b062a05e0908, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T09:59:45,590 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731751162662.ffe1934c280093c181c2b062a05e0908., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 , the old value is region=TestAcidGuarantees,,1731751162662.ffe1934c280093c181c2b062a05e0908., hostname=037c58a19ddb,37267,1731751077844, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffe1934c280093c181c2b062a05e0908, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:59:45,590 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] ipc.CallRunner(138): callId: 105 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43332 deadline: 1731751245583, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffe1934c280093c181c2b062a05e0908, server=037c58a19ddb,37267,1731751077844 2024-11-16T09:59:45,590 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731751162662.ffe1934c280093c181c2b062a05e0908., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffe1934c280093c181c2b062a05e0908, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:59:45,590 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731751162662.ffe1934c280093c181c2b062a05e0908., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 because the exception is null or not the one we care about 2024-11-16T09:59:45,590 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731751162662.ffe1934c280093c181c2b062a05e0908., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 , the old value is region=TestAcidGuarantees,,1731751162662.ffe1934c280093c181c2b062a05e0908., hostname=037c58a19ddb,37267,1731751077844, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffe1934c280093c181c2b062a05e0908, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:59:45,590 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731751162662.ffe1934c280093c181c2b062a05e0908., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffe1934c280093c181c2b062a05e0908, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:59:45,590 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731751162662.ffe1934c280093c181c2b062a05e0908., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 because the exception is null or not the one we care about 2024-11-16T09:59:45,598 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffe1934c280093c181c2b062a05e0908, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T09:59:45,598 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] ipc.CallRunner(138): callId: 108 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43418 deadline: 1731751245593, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffe1934c280093c181c2b062a05e0908, server=037c58a19ddb,37267,1731751077844 2024-11-16T09:59:45,599 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731751162662.ffe1934c280093c181c2b062a05e0908., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 , the old value is region=TestAcidGuarantees,,1731751162662.ffe1934c280093c181c2b062a05e0908., hostname=037c58a19ddb,37267,1731751077844, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffe1934c280093c181c2b062a05e0908, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:59:45,599 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731751162662.ffe1934c280093c181c2b062a05e0908., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffe1934c280093c181c2b062a05e0908, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:59:45,599 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731751162662.ffe1934c280093c181c2b062a05e0908., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 because the exception is null or not the one we care about 2024-11-16T09:59:45,608 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffe1934c280093c181c2b062a05e0908, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T09:59:45,608 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] ipc.CallRunner(138): callId: 107 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43402 deadline: 1731751245603, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffe1934c280093c181c2b062a05e0908, server=037c58a19ddb,37267,1731751077844 2024-11-16T09:59:45,608 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731751162662.ffe1934c280093c181c2b062a05e0908., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 , the old value is region=TestAcidGuarantees,,1731751162662.ffe1934c280093c181c2b062a05e0908., hostname=037c58a19ddb,37267,1731751077844, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffe1934c280093c181c2b062a05e0908, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:59:45,608 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731751162662.ffe1934c280093c181c2b062a05e0908., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffe1934c280093c181c2b062a05e0908, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:59:45,608 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731751162662.ffe1934c280093c181c2b062a05e0908., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 because the exception is null or not the one we care about 2024-11-16T09:59:45,773 INFO [Thread-1587 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1587. 2024-11-16T09:59:45,773 DEBUG [Thread-1587 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-16T09:59:45,773 DEBUG [Thread-1587 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T09:59:45,773 DEBUG [Thread-1587 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T09:59:45,774 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-16T09:59:45,776 INFO [Thread-1585 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1585. 2024-11-16T09:59:45,776 DEBUG [Thread-1585 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-16T09:59:45,777 DEBUG [Thread-1585 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T09:59:45,777 DEBUG [Thread-1585 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T09:59:45,777 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-16T09:59:45,777 INFO [Thread-1591 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1591. 2024-11-16T09:59:45,777 DEBUG [Thread-1591 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-16T09:59:45,777 DEBUG [Thread-1591 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T09:59:45,777 DEBUG [Thread-1591 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T09:59:45,777 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-16T09:59:45,781 INFO [Thread-1589 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1589. 2024-11-16T09:59:45,781 DEBUG [Thread-1589 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-16T09:59:45,781 DEBUG [Thread-1589 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T09:59:45,782 DEBUG [Thread-1589 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T09:59:45,782 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-16T09:59:45,784 INFO [Thread-1583 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1583. 2024-11-16T09:59:45,784 DEBUG [Thread-1583 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-16T09:59:45,784 DEBUG [Thread-1583 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T09:59:45,784 DEBUG [Thread-1583 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T09:59:45,784 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-16T09:59:45,863 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37877 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=109 2024-11-16T09:59:45,878 INFO [RS_FLUSH_OPERATIONS-regionserver/037c58a19ddb:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=287 (bloomFilter=true), to=hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/ffe1934c280093c181c2b062a05e0908/.tmp/C/92d4b326ce914ac591b817dc8a4ea75f 2024-11-16T09:59:45,882 DEBUG [RS_FLUSH_OPERATIONS-regionserver/037c58a19ddb:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/ffe1934c280093c181c2b062a05e0908/.tmp/A/fedbe5ffd79d430bbbbbeb61c1fe203e as hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/ffe1934c280093c181c2b062a05e0908/A/fedbe5ffd79d430bbbbbeb61c1fe203e 2024-11-16T09:59:45,885 INFO [RS_FLUSH_OPERATIONS-regionserver/037c58a19ddb:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/ffe1934c280093c181c2b062a05e0908/A/fedbe5ffd79d430bbbbbeb61c1fe203e, entries=150, sequenceid=287, filesize=30.5 K 2024-11-16T09:59:45,886 DEBUG [RS_FLUSH_OPERATIONS-regionserver/037c58a19ddb:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/ffe1934c280093c181c2b062a05e0908/.tmp/B/a6e336fc389343dab5c447e05135b43f as hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/ffe1934c280093c181c2b062a05e0908/B/a6e336fc389343dab5c447e05135b43f 2024-11-16T09:59:45,889 INFO [RS_FLUSH_OPERATIONS-regionserver/037c58a19ddb:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/ffe1934c280093c181c2b062a05e0908/B/a6e336fc389343dab5c447e05135b43f, entries=150, sequenceid=287, filesize=12.0 K 2024-11-16T09:59:45,890 DEBUG [RS_FLUSH_OPERATIONS-regionserver/037c58a19ddb:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/ffe1934c280093c181c2b062a05e0908/.tmp/C/92d4b326ce914ac591b817dc8a4ea75f as hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/ffe1934c280093c181c2b062a05e0908/C/92d4b326ce914ac591b817dc8a4ea75f 2024-11-16T09:59:45,892 INFO [RS_FLUSH_OPERATIONS-regionserver/037c58a19ddb:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/ffe1934c280093c181c2b062a05e0908/C/92d4b326ce914ac591b817dc8a4ea75f, entries=150, sequenceid=287, filesize=12.0 K 2024-11-16T09:59:45,893 INFO [RS_FLUSH_OPERATIONS-regionserver/037c58a19ddb:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.HRegion(3140): Finished flush of dataSize ~114.05 KB/116790, heapSize ~299.53 KB/306720, currentSize=87.22 KB/89310 for ffe1934c280093c181c2b062a05e0908 in 1256ms, sequenceid=287, compaction requested=false 2024-11-16T09:59:45,893 DEBUG [RS_FLUSH_OPERATIONS-regionserver/037c58a19ddb:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.HRegion(2603): Flush status journal for ffe1934c280093c181c2b062a05e0908: 2024-11-16T09:59:45,893 DEBUG [RS_FLUSH_OPERATIONS-regionserver/037c58a19ddb:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731751162662.ffe1934c280093c181c2b062a05e0908. 2024-11-16T09:59:45,893 DEBUG [RS_FLUSH_OPERATIONS-regionserver/037c58a19ddb:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=110 2024-11-16T09:59:45,894 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37877 {}] master.HMaster(4169): Remote procedure done, pid=110 2024-11-16T09:59:45,896 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=110, resume processing ppid=109 2024-11-16T09:59:45,896 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=110, ppid=109, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.1770 sec 2024-11-16T09:59:45,897 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=109, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=109, table=TestAcidGuarantees in 2.1820 sec 2024-11-16T09:59:46,104 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] regionserver.HRegion(8855): Flush requested on ffe1934c280093c181c2b062a05e0908 2024-11-16T09:59:46,104 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-16T09:59:46,104 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing ffe1934c280093c181c2b062a05e0908 3/3 column families, dataSize=107.34 KB heapSize=282 KB 2024-11-16T09:59:46,104 INFO [Thread-1576 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1576. 2024-11-16T09:59:46,105 INFO [Thread-1580 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1580. 2024-11-16T09:59:46,105 DEBUG [Thread-1576 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-16T09:59:46,105 DEBUG [Thread-1576 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T09:59:46,105 DEBUG [Thread-1580 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-16T09:59:46,105 DEBUG [Thread-1580 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T09:59:46,105 DEBUG [Thread-1576 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T09:59:46,105 DEBUG [Thread-1580 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T09:59:46,105 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-16T09:59:46,105 INFO [Thread-1578 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1578. 2024-11-16T09:59:46,105 DEBUG [Thread-1578 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-16T09:59:46,105 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-16T09:59:46,105 DEBUG [Thread-1578 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T09:59:46,105 DEBUG [Thread-1578 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T09:59:46,105 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-16T09:59:46,106 INFO [Thread-1572 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1572. 2024-11-16T09:59:46,106 DEBUG [Thread-1572 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-16T09:59:46,106 DEBUG [Thread-1572 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T09:59:46,106 DEBUG [Thread-1572 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T09:59:46,106 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-16T09:59:46,111 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411167b12dd1e49d4494db2d8453572b63c0f_ffe1934c280093c181c2b062a05e0908 is 50, key is test_row_0/A:col10/1731751184932/Put/seqid=0 2024-11-16T09:59:46,114 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37591 is added to blk_1073742255_1431 (size=12454) 2024-11-16T09:59:46,114 INFO [Thread-1574 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1574. 2024-11-16T09:59:46,114 DEBUG [Thread-1574 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-16T09:59:46,114 DEBUG [Thread-1574 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T09:59:46,114 DEBUG [Thread-1574 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T09:59:46,114 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-16T09:59:46,515 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T09:59:46,518 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411167b12dd1e49d4494db2d8453572b63c0f_ffe1934c280093c181c2b062a05e0908 to hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411167b12dd1e49d4494db2d8453572b63c0f_ffe1934c280093c181c2b062a05e0908 2024-11-16T09:59:46,518 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/ffe1934c280093c181c2b062a05e0908/.tmp/A/24bec8bf9c1c40a5afbc4caca203cc56, store: [table=TestAcidGuarantees family=A region=ffe1934c280093c181c2b062a05e0908] 2024-11-16T09:59:46,519 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/ffe1934c280093c181c2b062a05e0908/.tmp/A/24bec8bf9c1c40a5afbc4caca203cc56 is 175, key is test_row_0/A:col10/1731751184932/Put/seqid=0 2024-11-16T09:59:46,522 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37591 is added to blk_1073742256_1432 (size=31255) 2024-11-16T09:59:46,923 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=310, memsize=38.0 K, hasBloomFilter=true, into tmp file hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/ffe1934c280093c181c2b062a05e0908/.tmp/A/24bec8bf9c1c40a5afbc4caca203cc56 2024-11-16T09:59:46,930 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/ffe1934c280093c181c2b062a05e0908/.tmp/B/97d1a274f611400ab3a9315e75ebea67 is 50, key is test_row_0/B:col10/1731751184932/Put/seqid=0 2024-11-16T09:59:46,933 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37591 is added to blk_1073742257_1433 (size=12301) 2024-11-16T09:59:47,335 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=310 (bloomFilter=true), to=hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/ffe1934c280093c181c2b062a05e0908/.tmp/B/97d1a274f611400ab3a9315e75ebea67 2024-11-16T09:59:47,343 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/ffe1934c280093c181c2b062a05e0908/.tmp/C/a0dae046833e43a299803c1a8906fdcb is 50, key is test_row_0/C:col10/1731751184932/Put/seqid=0 2024-11-16T09:59:47,353 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37591 is added to blk_1073742258_1434 (size=12301) 2024-11-16T09:59:47,754 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=310 (bloomFilter=true), to=hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/ffe1934c280093c181c2b062a05e0908/.tmp/C/a0dae046833e43a299803c1a8906fdcb 2024-11-16T09:59:47,758 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/ffe1934c280093c181c2b062a05e0908/.tmp/A/24bec8bf9c1c40a5afbc4caca203cc56 as hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/ffe1934c280093c181c2b062a05e0908/A/24bec8bf9c1c40a5afbc4caca203cc56 2024-11-16T09:59:47,761 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/ffe1934c280093c181c2b062a05e0908/A/24bec8bf9c1c40a5afbc4caca203cc56, entries=150, sequenceid=310, filesize=30.5 K 2024-11-16T09:59:47,762 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/ffe1934c280093c181c2b062a05e0908/.tmp/B/97d1a274f611400ab3a9315e75ebea67 as hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/ffe1934c280093c181c2b062a05e0908/B/97d1a274f611400ab3a9315e75ebea67 2024-11-16T09:59:47,765 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/ffe1934c280093c181c2b062a05e0908/B/97d1a274f611400ab3a9315e75ebea67, entries=150, sequenceid=310, filesize=12.0 K 2024-11-16T09:59:47,765 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/ffe1934c280093c181c2b062a05e0908/.tmp/C/a0dae046833e43a299803c1a8906fdcb as hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/ffe1934c280093c181c2b062a05e0908/C/a0dae046833e43a299803c1a8906fdcb 2024-11-16T09:59:47,768 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/ffe1934c280093c181c2b062a05e0908/C/a0dae046833e43a299803c1a8906fdcb, entries=150, sequenceid=310, filesize=12.0 K 2024-11-16T09:59:47,769 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~114.05 KB/116790, heapSize ~299.53 KB/306720, currentSize=6.71 KB/6870 for ffe1934c280093c181c2b062a05e0908 in 1665ms, sequenceid=310, compaction requested=true 2024-11-16T09:59:47,769 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for ffe1934c280093c181c2b062a05e0908: 2024-11-16T09:59:47,769 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store ffe1934c280093c181c2b062a05e0908:A, priority=-2147483648, current under compaction store size is 1 2024-11-16T09:59:47,769 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-16T09:59:47,769 DEBUG [RS:0;037c58a19ddb:37267-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-16T09:59:47,769 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store ffe1934c280093c181c2b062a05e0908:B, priority=-2147483648, current under compaction store size is 2 2024-11-16T09:59:47,769 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-16T09:59:47,769 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store ffe1934c280093c181c2b062a05e0908:C, priority=-2147483648, current under compaction store size is 3 2024-11-16T09:59:47,769 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-16T09:59:47,769 DEBUG [RS:0;037c58a19ddb:37267-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-16T09:59:47,770 DEBUG [RS:0;037c58a19ddb:37267-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 94413 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-16T09:59:47,770 DEBUG [RS:0;037c58a19ddb:37267-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37551 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-16T09:59:47,770 DEBUG [RS:0;037c58a19ddb:37267-shortCompactions-0 {}] regionserver.HStore(1541): ffe1934c280093c181c2b062a05e0908/A is initiating minor compaction (all files) 2024-11-16T09:59:47,770 DEBUG [RS:0;037c58a19ddb:37267-longCompactions-0 {}] regionserver.HStore(1541): ffe1934c280093c181c2b062a05e0908/B is initiating minor compaction (all files) 2024-11-16T09:59:47,770 INFO [RS:0;037c58a19ddb:37267-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of ffe1934c280093c181c2b062a05e0908/A in TestAcidGuarantees,,1731751162662.ffe1934c280093c181c2b062a05e0908. 2024-11-16T09:59:47,770 INFO [RS:0;037c58a19ddb:37267-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of ffe1934c280093c181c2b062a05e0908/B in TestAcidGuarantees,,1731751162662.ffe1934c280093c181c2b062a05e0908. 2024-11-16T09:59:47,770 INFO [RS:0;037c58a19ddb:37267-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/ffe1934c280093c181c2b062a05e0908/A/8f7d13fd125a4e48ba6db638d37dd1d3, hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/ffe1934c280093c181c2b062a05e0908/A/fedbe5ffd79d430bbbbbeb61c1fe203e, hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/ffe1934c280093c181c2b062a05e0908/A/24bec8bf9c1c40a5afbc4caca203cc56] into tmpdir=hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/ffe1934c280093c181c2b062a05e0908/.tmp, totalSize=92.2 K 2024-11-16T09:59:47,770 INFO [RS:0;037c58a19ddb:37267-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/ffe1934c280093c181c2b062a05e0908/B/2bd7c66293374d64944f326ac038a69b, hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/ffe1934c280093c181c2b062a05e0908/B/a6e336fc389343dab5c447e05135b43f, hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/ffe1934c280093c181c2b062a05e0908/B/97d1a274f611400ab3a9315e75ebea67] into tmpdir=hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/ffe1934c280093c181c2b062a05e0908/.tmp, totalSize=36.7 K 2024-11-16T09:59:47,770 INFO [RS:0;037c58a19ddb:37267-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731751162662.ffe1934c280093c181c2b062a05e0908. 2024-11-16T09:59:47,770 DEBUG [RS:0;037c58a19ddb:37267-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731751162662.ffe1934c280093c181c2b062a05e0908. files: [hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/ffe1934c280093c181c2b062a05e0908/A/8f7d13fd125a4e48ba6db638d37dd1d3, hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/ffe1934c280093c181c2b062a05e0908/A/fedbe5ffd79d430bbbbbeb61c1fe203e, hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/ffe1934c280093c181c2b062a05e0908/A/24bec8bf9c1c40a5afbc4caca203cc56] 2024-11-16T09:59:47,770 DEBUG [RS:0;037c58a19ddb:37267-longCompactions-0 {}] compactions.Compactor(225): Compacting 2bd7c66293374d64944f326ac038a69b, keycount=150, bloomtype=ROW, size=12.6 K, encoding=NONE, compression=NONE, seqNum=267, earliestPutTs=1731751183067 2024-11-16T09:59:47,770 DEBUG [RS:0;037c58a19ddb:37267-shortCompactions-0 {}] compactions.Compactor(225): Compacting 8f7d13fd125a4e48ba6db638d37dd1d3, keycount=150, bloomtype=ROW, size=31.2 K, encoding=NONE, compression=NONE, seqNum=267, earliestPutTs=1731751183067 2024-11-16T09:59:47,770 DEBUG [RS:0;037c58a19ddb:37267-longCompactions-0 {}] compactions.Compactor(225): Compacting a6e336fc389343dab5c447e05135b43f, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=287, earliestPutTs=1731751183721 2024-11-16T09:59:47,770 DEBUG [RS:0;037c58a19ddb:37267-shortCompactions-0 {}] compactions.Compactor(225): Compacting fedbe5ffd79d430bbbbbeb61c1fe203e, keycount=150, bloomtype=ROW, size=30.5 K, encoding=NONE, compression=NONE, seqNum=287, earliestPutTs=1731751183721 2024-11-16T09:59:47,771 DEBUG [RS:0;037c58a19ddb:37267-shortCompactions-0 {}] compactions.Compactor(225): Compacting 24bec8bf9c1c40a5afbc4caca203cc56, keycount=150, bloomtype=ROW, size=30.5 K, encoding=NONE, compression=NONE, seqNum=310, earliestPutTs=1731751184932 2024-11-16T09:59:47,771 DEBUG [RS:0;037c58a19ddb:37267-longCompactions-0 {}] compactions.Compactor(225): Compacting 97d1a274f611400ab3a9315e75ebea67, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=310, earliestPutTs=1731751184932 2024-11-16T09:59:47,777 INFO [RS:0;037c58a19ddb:37267-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): ffe1934c280093c181c2b062a05e0908#B#compaction#365 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-16T09:59:47,778 DEBUG [RS:0;037c58a19ddb:37267-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/ffe1934c280093c181c2b062a05e0908/.tmp/B/5f9e7a026fb94fa19c5eb278dead9713 is 50, key is test_row_0/B:col10/1731751184932/Put/seqid=0 2024-11-16T09:59:47,781 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37591 is added to blk_1073742259_1435 (size=13051) 2024-11-16T09:59:47,783 INFO [RS:0;037c58a19ddb:37267-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=ffe1934c280093c181c2b062a05e0908] 2024-11-16T09:59:47,785 DEBUG [RS:0;037c58a19ddb:37267-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e202411166f49343d38a34e33aa8640fe730047f2_ffe1934c280093c181c2b062a05e0908 store=[table=TestAcidGuarantees family=A region=ffe1934c280093c181c2b062a05e0908] 2024-11-16T09:59:47,813 DEBUG [RS:0;037c58a19ddb:37267-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e202411166f49343d38a34e33aa8640fe730047f2_ffe1934c280093c181c2b062a05e0908, store=[table=TestAcidGuarantees family=A region=ffe1934c280093c181c2b062a05e0908] 2024-11-16T09:59:47,813 DEBUG [RS:0;037c58a19ddb:37267-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411166f49343d38a34e33aa8640fe730047f2_ffe1934c280093c181c2b062a05e0908 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=ffe1934c280093c181c2b062a05e0908] 2024-11-16T09:59:47,817 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37591 is added to blk_1073742260_1436 (size=4469) 2024-11-16T09:59:47,883 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37877 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=109 2024-11-16T09:59:47,884 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-16T09:59:47,884 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(392): Finished test. Writers: 2024-11-16T09:59:47,884 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 48 2024-11-16T09:59:47,884 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 50 2024-11-16T09:59:47,884 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 50 2024-11-16T09:59:47,884 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 51 2024-11-16T09:59:47,884 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 42 2024-11-16T09:59:47,884 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(396): Readers: 2024-11-16T09:59:47,884 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(400): Scanners: 2024-11-16T09:59:47,884 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1403 2024-11-16T09:59:47,884 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 4206 rows 2024-11-16T09:59:47,884 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1404 2024-11-16T09:59:47,884 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 4209 rows 2024-11-16T09:59:47,884 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1400 2024-11-16T09:59:47,884 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 4197 rows 2024-11-16T09:59:47,884 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1401 2024-11-16T09:59:47,884 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 4197 rows 2024-11-16T09:59:47,884 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1408 2024-11-16T09:59:47,884 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 4222 rows 2024-11-16T09:59:47,884 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-16T09:59:47,884 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool.doWork(AcidGuaranteesTestTool.java:131) at org.apache.hadoop.hbase.util.AbstractHBaseTool.run(AbstractHBaseTool.java:150) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.runTestAtomicity(AcidGuaranteesTestBase.java:102) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.testMobScanAtomicity(AcidGuaranteesTestBase.java:127) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-16T09:59:47,884 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T09:59:47,885 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T09:59:47,885 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-16T09:59:47,886 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37877 {}] master.HMaster$13(2820): Client=jenkins//172.17.0.2 disable TestAcidGuarantees 2024-11-16T09:59:47,887 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37877 {}] procedure2.ProcedureExecutor(1139): Stored pid=111, state=RUNNABLE:DISABLE_TABLE_PREPARE, hasLock=false; DisableTableProcedure table=TestAcidGuarantees 2024-11-16T09:59:47,889 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37877 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=111 2024-11-16T09:59:47,890 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731751187889"}]},"ts":"1731751187889"} 2024-11-16T09:59:47,891 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLING in hbase:meta 2024-11-16T09:59:47,891 INFO [PEWorker-3 {}] procedure.DisableTableProcedure(284): Set TestAcidGuarantees to state=DISABLING 2024-11-16T09:59:47,892 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=112, ppid=111, state=RUNNABLE:CLOSE_TABLE_REGIONS_SCHEDULE, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees}] 2024-11-16T09:59:47,894 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=113, ppid=112, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=ffe1934c280093c181c2b062a05e0908, UNASSIGN}] 2024-11-16T09:59:47,894 INFO [PEWorker-5 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=113, ppid=112, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=ffe1934c280093c181c2b062a05e0908, UNASSIGN 2024-11-16T09:59:47,895 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=113 updating hbase:meta row=ffe1934c280093c181c2b062a05e0908, regionState=CLOSING, regionLocation=037c58a19ddb,37267,1731751077844 2024-11-16T09:59:47,896 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=113, ppid=112, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=ffe1934c280093c181c2b062a05e0908, UNASSIGN because future has completed 2024-11-16T09:59:47,897 DEBUG [PEWorker-1 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-11-16T09:59:47,897 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=114, ppid=113, state=RUNNABLE, hasLock=false; CloseRegionProcedure ffe1934c280093c181c2b062a05e0908, server=037c58a19ddb,37267,1731751077844}] 2024-11-16T09:59:47,993 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37877 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=111 2024-11-16T09:59:48,049 INFO [RS_CLOSE_REGION-regionserver/037c58a19ddb:0-0 {event_type=M_RS_CLOSE_REGION, pid=114}] handler.UnassignRegionHandler(122): Close ffe1934c280093c181c2b062a05e0908 2024-11-16T09:59:48,049 DEBUG [RS_CLOSE_REGION-regionserver/037c58a19ddb:0-0 {event_type=M_RS_CLOSE_REGION, pid=114}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-11-16T09:59:48,050 DEBUG [RS_CLOSE_REGION-regionserver/037c58a19ddb:0-0 {event_type=M_RS_CLOSE_REGION, pid=114}] regionserver.HRegion(1722): Closing ffe1934c280093c181c2b062a05e0908, disabling compactions & flushes 2024-11-16T09:59:48,050 DEBUG [RS_CLOSE_REGION-regionserver/037c58a19ddb:0-0 {event_type=M_RS_CLOSE_REGION, pid=114}] regionserver.HRegion(1993): waiting for 2 compactions to complete for region TestAcidGuarantees,,1731751162662.ffe1934c280093c181c2b062a05e0908. 2024-11-16T09:59:48,186 DEBUG [RS:0;037c58a19ddb:37267-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/ffe1934c280093c181c2b062a05e0908/.tmp/B/5f9e7a026fb94fa19c5eb278dead9713 as hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/ffe1934c280093c181c2b062a05e0908/B/5f9e7a026fb94fa19c5eb278dead9713 2024-11-16T09:59:48,191 INFO [RS:0;037c58a19ddb:37267-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in ffe1934c280093c181c2b062a05e0908/B of ffe1934c280093c181c2b062a05e0908 into 5f9e7a026fb94fa19c5eb278dead9713(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-16T09:59:48,191 DEBUG [RS:0;037c58a19ddb:37267-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for ffe1934c280093c181c2b062a05e0908: 2024-11-16T09:59:48,191 INFO [RS:0;037c58a19ddb:37267-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731751162662.ffe1934c280093c181c2b062a05e0908., storeName=ffe1934c280093c181c2b062a05e0908/B, priority=13, startTime=1731751187769; duration=0sec 2024-11-16T09:59:48,191 DEBUG [RS:0;037c58a19ddb:37267-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-16T09:59:48,191 DEBUG [RS:0;037c58a19ddb:37267-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: ffe1934c280093c181c2b062a05e0908:B 2024-11-16T09:59:48,191 DEBUG [RS:0;037c58a19ddb:37267-longCompactions-0 {}] regionserver.CompactSplit(450): Not compacting TestAcidGuarantees,,1731751162662.ffe1934c280093c181c2b062a05e0908. because compaction request was cancelled 2024-11-16T09:59:48,191 DEBUG [RS:0;037c58a19ddb:37267-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: ffe1934c280093c181c2b062a05e0908:C 2024-11-16T09:59:48,202 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37877 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=111 2024-11-16T09:59:48,218 INFO [RS:0;037c58a19ddb:37267-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): ffe1934c280093c181c2b062a05e0908#A#compaction#366 average throughput is 0.06 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-16T09:59:48,218 DEBUG [RS:0;037c58a19ddb:37267-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/ffe1934c280093c181c2b062a05e0908/.tmp/A/fb3563ac534d4a618da4ec8b60e77b17 is 175, key is test_row_0/A:col10/1731751184932/Put/seqid=0 2024-11-16T09:59:48,221 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37591 is added to blk_1073742261_1437 (size=32005) 2024-11-16T09:59:48,513 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37877 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=111 2024-11-16T09:59:48,626 DEBUG [RS:0;037c58a19ddb:37267-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/ffe1934c280093c181c2b062a05e0908/.tmp/A/fb3563ac534d4a618da4ec8b60e77b17 as hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/ffe1934c280093c181c2b062a05e0908/A/fb3563ac534d4a618da4ec8b60e77b17 2024-11-16T09:59:48,630 INFO [RS:0;037c58a19ddb:37267-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in ffe1934c280093c181c2b062a05e0908/A of ffe1934c280093c181c2b062a05e0908 into fb3563ac534d4a618da4ec8b60e77b17(size=31.3 K), total size for store is 31.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-16T09:59:48,630 DEBUG [RS:0;037c58a19ddb:37267-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for ffe1934c280093c181c2b062a05e0908: 2024-11-16T09:59:48,630 INFO [RS:0;037c58a19ddb:37267-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731751162662.ffe1934c280093c181c2b062a05e0908., storeName=ffe1934c280093c181c2b062a05e0908/A, priority=13, startTime=1731751187769; duration=0sec 2024-11-16T09:59:48,630 INFO [RS_CLOSE_REGION-regionserver/037c58a19ddb:0-0 {event_type=M_RS_CLOSE_REGION, pid=114}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1731751162662.ffe1934c280093c181c2b062a05e0908. 2024-11-16T09:59:48,630 DEBUG [RS:0;037c58a19ddb:37267-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-16T09:59:48,630 DEBUG [RS_CLOSE_REGION-regionserver/037c58a19ddb:0-0 {event_type=M_RS_CLOSE_REGION, pid=114}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1731751162662.ffe1934c280093c181c2b062a05e0908. 2024-11-16T09:59:48,630 DEBUG [RS:0;037c58a19ddb:37267-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: ffe1934c280093c181c2b062a05e0908:A 2024-11-16T09:59:48,630 DEBUG [RS_CLOSE_REGION-regionserver/037c58a19ddb:0-0 {event_type=M_RS_CLOSE_REGION, pid=114}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1731751162662.ffe1934c280093c181c2b062a05e0908. after waiting 0 ms 2024-11-16T09:59:48,630 DEBUG [RS_CLOSE_REGION-regionserver/037c58a19ddb:0-0 {event_type=M_RS_CLOSE_REGION, pid=114}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1731751162662.ffe1934c280093c181c2b062a05e0908. 2024-11-16T09:59:48,630 INFO [RS_CLOSE_REGION-regionserver/037c58a19ddb:0-0 {event_type=M_RS_CLOSE_REGION, pid=114}] regionserver.HRegion(2902): Flushing ffe1934c280093c181c2b062a05e0908 3/3 column families, dataSize=6.71 KB heapSize=18.33 KB 2024-11-16T09:59:48,634 DEBUG [RS_CLOSE_REGION-regionserver/037c58a19ddb:0-0 {event_type=M_RS_CLOSE_REGION, pid=114}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111611a0e4b311e040019efb82a03d8d338f_ffe1934c280093c181c2b062a05e0908 is 50, key is test_row_0/A:col10/1731751186113/Put/seqid=0 2024-11-16T09:59:48,638 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37591 is added to blk_1073742262_1438 (size=7374) 2024-11-16T09:59:49,023 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37877 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=111 2024-11-16T09:59:49,038 DEBUG [RS_CLOSE_REGION-regionserver/037c58a19ddb:0-0 {event_type=M_RS_CLOSE_REGION, pid=114}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T09:59:49,041 INFO [RS_CLOSE_REGION-regionserver/037c58a19ddb:0-0 {event_type=M_RS_CLOSE_REGION, pid=114}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111611a0e4b311e040019efb82a03d8d338f_ffe1934c280093c181c2b062a05e0908 to hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111611a0e4b311e040019efb82a03d8d338f_ffe1934c280093c181c2b062a05e0908 2024-11-16T09:59:49,042 DEBUG [RS_CLOSE_REGION-regionserver/037c58a19ddb:0-0 {event_type=M_RS_CLOSE_REGION, pid=114}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/ffe1934c280093c181c2b062a05e0908/.tmp/A/a8b81e40e710410394660fce77f9a304, store: [table=TestAcidGuarantees family=A region=ffe1934c280093c181c2b062a05e0908] 2024-11-16T09:59:49,043 DEBUG [RS_CLOSE_REGION-regionserver/037c58a19ddb:0-0 {event_type=M_RS_CLOSE_REGION, pid=114}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/ffe1934c280093c181c2b062a05e0908/.tmp/A/a8b81e40e710410394660fce77f9a304 is 175, key is test_row_0/A:col10/1731751186113/Put/seqid=0 2024-11-16T09:59:49,046 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37591 is added to blk_1073742263_1439 (size=13865) 2024-11-16T09:59:49,446 INFO [RS_CLOSE_REGION-regionserver/037c58a19ddb:0-0 {event_type=M_RS_CLOSE_REGION, pid=114}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=316, memsize=2.2 K, hasBloomFilter=true, into tmp file hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/ffe1934c280093c181c2b062a05e0908/.tmp/A/a8b81e40e710410394660fce77f9a304 2024-11-16T09:59:49,452 DEBUG [RS_CLOSE_REGION-regionserver/037c58a19ddb:0-0 {event_type=M_RS_CLOSE_REGION, pid=114}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/ffe1934c280093c181c2b062a05e0908/.tmp/B/739d447954a74addafe8c957d46f719f is 50, key is test_row_0/B:col10/1731751186113/Put/seqid=0 2024-11-16T09:59:49,455 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37591 is added to blk_1073742264_1440 (size=7415) 2024-11-16T09:59:49,856 INFO [RS_CLOSE_REGION-regionserver/037c58a19ddb:0-0 {event_type=M_RS_CLOSE_REGION, pid=114}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=2.24 KB at sequenceid=316 (bloomFilter=true), to=hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/ffe1934c280093c181c2b062a05e0908/.tmp/B/739d447954a74addafe8c957d46f719f 2024-11-16T09:59:49,862 DEBUG [RS_CLOSE_REGION-regionserver/037c58a19ddb:0-0 {event_type=M_RS_CLOSE_REGION, pid=114}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/ffe1934c280093c181c2b062a05e0908/.tmp/C/1cbc6d0426af4c569b8949b7f82a3b73 is 50, key is test_row_0/C:col10/1731751186113/Put/seqid=0 2024-11-16T09:59:49,865 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37591 is added to blk_1073742265_1441 (size=7415) 2024-11-16T09:59:50,033 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37877 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=111 2024-11-16T09:59:50,266 INFO [RS_CLOSE_REGION-regionserver/037c58a19ddb:0-0 {event_type=M_RS_CLOSE_REGION, pid=114}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=2.24 KB at sequenceid=316 (bloomFilter=true), to=hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/ffe1934c280093c181c2b062a05e0908/.tmp/C/1cbc6d0426af4c569b8949b7f82a3b73 2024-11-16T09:59:50,269 DEBUG [RS_CLOSE_REGION-regionserver/037c58a19ddb:0-0 {event_type=M_RS_CLOSE_REGION, pid=114}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/ffe1934c280093c181c2b062a05e0908/.tmp/A/a8b81e40e710410394660fce77f9a304 as hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/ffe1934c280093c181c2b062a05e0908/A/a8b81e40e710410394660fce77f9a304 2024-11-16T09:59:50,272 INFO [RS_CLOSE_REGION-regionserver/037c58a19ddb:0-0 {event_type=M_RS_CLOSE_REGION, pid=114}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/ffe1934c280093c181c2b062a05e0908/A/a8b81e40e710410394660fce77f9a304, entries=50, sequenceid=316, filesize=13.5 K 2024-11-16T09:59:50,273 DEBUG [RS_CLOSE_REGION-regionserver/037c58a19ddb:0-0 {event_type=M_RS_CLOSE_REGION, pid=114}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/ffe1934c280093c181c2b062a05e0908/.tmp/B/739d447954a74addafe8c957d46f719f as hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/ffe1934c280093c181c2b062a05e0908/B/739d447954a74addafe8c957d46f719f 2024-11-16T09:59:50,276 INFO [RS_CLOSE_REGION-regionserver/037c58a19ddb:0-0 {event_type=M_RS_CLOSE_REGION, pid=114}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/ffe1934c280093c181c2b062a05e0908/B/739d447954a74addafe8c957d46f719f, entries=50, sequenceid=316, filesize=7.2 K 2024-11-16T09:59:50,276 DEBUG [RS_CLOSE_REGION-regionserver/037c58a19ddb:0-0 {event_type=M_RS_CLOSE_REGION, pid=114}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/ffe1934c280093c181c2b062a05e0908/.tmp/C/1cbc6d0426af4c569b8949b7f82a3b73 as hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/ffe1934c280093c181c2b062a05e0908/C/1cbc6d0426af4c569b8949b7f82a3b73 2024-11-16T09:59:50,279 INFO [RS_CLOSE_REGION-regionserver/037c58a19ddb:0-0 {event_type=M_RS_CLOSE_REGION, pid=114}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/ffe1934c280093c181c2b062a05e0908/C/1cbc6d0426af4c569b8949b7f82a3b73, entries=50, sequenceid=316, filesize=7.2 K 2024-11-16T09:59:50,280 INFO [RS_CLOSE_REGION-regionserver/037c58a19ddb:0-0 {event_type=M_RS_CLOSE_REGION, pid=114}] regionserver.HRegion(3140): Finished flush of dataSize ~6.71 KB/6870, heapSize ~18.28 KB/18720, currentSize=0 B/0 for ffe1934c280093c181c2b062a05e0908 in 1649ms, sequenceid=316, compaction requested=true 2024-11-16T09:59:50,280 DEBUG [StoreCloser-TestAcidGuarantees,,1731751162662.ffe1934c280093c181c2b062a05e0908.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/ffe1934c280093c181c2b062a05e0908/A/2d827c9c04a74475be2edfc99ef61e4e, hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/ffe1934c280093c181c2b062a05e0908/A/32f240e1f90d4f1c826f4682f47a04f0, hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/ffe1934c280093c181c2b062a05e0908/A/76773910668b4f61841d0bde508173c5, hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/ffe1934c280093c181c2b062a05e0908/A/4939aa1e225f4585a4c8405038aea298, hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/ffe1934c280093c181c2b062a05e0908/A/6d3471f357e64c24aa32cc3a11602671, hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/ffe1934c280093c181c2b062a05e0908/A/70b600411963462196c33bf15fa80c47, hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/ffe1934c280093c181c2b062a05e0908/A/f9a4ab02427a4739bf321279a7bcb212, hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/ffe1934c280093c181c2b062a05e0908/A/b94361fb6b4448359e6a3cb995534553, hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/ffe1934c280093c181c2b062a05e0908/A/fc3e9b5bbfd24a0480896ecf8db38f28, hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/ffe1934c280093c181c2b062a05e0908/A/4f3739c53ae949579bcbf3fa1d7bcc34, hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/ffe1934c280093c181c2b062a05e0908/A/4de463595afb42a0b38dc1e8ddeca7b7, hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/ffe1934c280093c181c2b062a05e0908/A/ee2723f2d8864f99892f888682a906e1, hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/ffe1934c280093c181c2b062a05e0908/A/3160dd6fd84e4b82b402bd46540cc5ed, hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/ffe1934c280093c181c2b062a05e0908/A/496b53efeb1648a1ae19d0a8097b6c2b, hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/ffe1934c280093c181c2b062a05e0908/A/8e8ef152dca44dd8a3f29446df0b9790, hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/ffe1934c280093c181c2b062a05e0908/A/9e8350a74f8a4165b66b073a641f4577, hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/ffe1934c280093c181c2b062a05e0908/A/2bbc77228e7446d4826467c6853a1981, hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/ffe1934c280093c181c2b062a05e0908/A/dac023d72c9a467cb301eb07f30695ca, hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/ffe1934c280093c181c2b062a05e0908/A/e0361185f7ea491a8b4f44995bc8f102, hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/ffe1934c280093c181c2b062a05e0908/A/8f7d13fd125a4e48ba6db638d37dd1d3, hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/ffe1934c280093c181c2b062a05e0908/A/fedbe5ffd79d430bbbbbeb61c1fe203e, hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/ffe1934c280093c181c2b062a05e0908/A/24bec8bf9c1c40a5afbc4caca203cc56] to archive 2024-11-16T09:59:50,281 DEBUG [StoreCloser-TestAcidGuarantees,,1731751162662.ffe1934c280093c181c2b062a05e0908.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-16T09:59:50,282 DEBUG [StoreCloser-TestAcidGuarantees,,1731751162662.ffe1934c280093c181c2b062a05e0908.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/ffe1934c280093c181c2b062a05e0908/A/2d827c9c04a74475be2edfc99ef61e4e to hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/archive/data/default/TestAcidGuarantees/ffe1934c280093c181c2b062a05e0908/A/2d827c9c04a74475be2edfc99ef61e4e 2024-11-16T09:59:50,283 DEBUG [StoreCloser-TestAcidGuarantees,,1731751162662.ffe1934c280093c181c2b062a05e0908.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/ffe1934c280093c181c2b062a05e0908/A/32f240e1f90d4f1c826f4682f47a04f0 to hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/archive/data/default/TestAcidGuarantees/ffe1934c280093c181c2b062a05e0908/A/32f240e1f90d4f1c826f4682f47a04f0 2024-11-16T09:59:50,284 DEBUG [StoreCloser-TestAcidGuarantees,,1731751162662.ffe1934c280093c181c2b062a05e0908.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/ffe1934c280093c181c2b062a05e0908/A/76773910668b4f61841d0bde508173c5 to hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/archive/data/default/TestAcidGuarantees/ffe1934c280093c181c2b062a05e0908/A/76773910668b4f61841d0bde508173c5 2024-11-16T09:59:50,285 DEBUG [StoreCloser-TestAcidGuarantees,,1731751162662.ffe1934c280093c181c2b062a05e0908.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/ffe1934c280093c181c2b062a05e0908/A/4939aa1e225f4585a4c8405038aea298 to hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/archive/data/default/TestAcidGuarantees/ffe1934c280093c181c2b062a05e0908/A/4939aa1e225f4585a4c8405038aea298 2024-11-16T09:59:50,285 DEBUG [StoreCloser-TestAcidGuarantees,,1731751162662.ffe1934c280093c181c2b062a05e0908.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/ffe1934c280093c181c2b062a05e0908/A/6d3471f357e64c24aa32cc3a11602671 to hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/archive/data/default/TestAcidGuarantees/ffe1934c280093c181c2b062a05e0908/A/6d3471f357e64c24aa32cc3a11602671 2024-11-16T09:59:50,286 DEBUG [StoreCloser-TestAcidGuarantees,,1731751162662.ffe1934c280093c181c2b062a05e0908.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/ffe1934c280093c181c2b062a05e0908/A/70b600411963462196c33bf15fa80c47 to hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/archive/data/default/TestAcidGuarantees/ffe1934c280093c181c2b062a05e0908/A/70b600411963462196c33bf15fa80c47 2024-11-16T09:59:50,287 DEBUG [StoreCloser-TestAcidGuarantees,,1731751162662.ffe1934c280093c181c2b062a05e0908.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/ffe1934c280093c181c2b062a05e0908/A/f9a4ab02427a4739bf321279a7bcb212 to hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/archive/data/default/TestAcidGuarantees/ffe1934c280093c181c2b062a05e0908/A/f9a4ab02427a4739bf321279a7bcb212 2024-11-16T09:59:50,288 DEBUG [StoreCloser-TestAcidGuarantees,,1731751162662.ffe1934c280093c181c2b062a05e0908.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/ffe1934c280093c181c2b062a05e0908/A/b94361fb6b4448359e6a3cb995534553 to hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/archive/data/default/TestAcidGuarantees/ffe1934c280093c181c2b062a05e0908/A/b94361fb6b4448359e6a3cb995534553 2024-11-16T09:59:50,288 DEBUG [StoreCloser-TestAcidGuarantees,,1731751162662.ffe1934c280093c181c2b062a05e0908.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/ffe1934c280093c181c2b062a05e0908/A/fc3e9b5bbfd24a0480896ecf8db38f28 to hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/archive/data/default/TestAcidGuarantees/ffe1934c280093c181c2b062a05e0908/A/fc3e9b5bbfd24a0480896ecf8db38f28 2024-11-16T09:59:50,289 DEBUG [StoreCloser-TestAcidGuarantees,,1731751162662.ffe1934c280093c181c2b062a05e0908.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/ffe1934c280093c181c2b062a05e0908/A/4f3739c53ae949579bcbf3fa1d7bcc34 to hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/archive/data/default/TestAcidGuarantees/ffe1934c280093c181c2b062a05e0908/A/4f3739c53ae949579bcbf3fa1d7bcc34 2024-11-16T09:59:50,290 DEBUG [StoreCloser-TestAcidGuarantees,,1731751162662.ffe1934c280093c181c2b062a05e0908.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/ffe1934c280093c181c2b062a05e0908/A/4de463595afb42a0b38dc1e8ddeca7b7 to hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/archive/data/default/TestAcidGuarantees/ffe1934c280093c181c2b062a05e0908/A/4de463595afb42a0b38dc1e8ddeca7b7 2024-11-16T09:59:50,291 DEBUG [StoreCloser-TestAcidGuarantees,,1731751162662.ffe1934c280093c181c2b062a05e0908.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/ffe1934c280093c181c2b062a05e0908/A/ee2723f2d8864f99892f888682a906e1 to hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/archive/data/default/TestAcidGuarantees/ffe1934c280093c181c2b062a05e0908/A/ee2723f2d8864f99892f888682a906e1 2024-11-16T09:59:50,292 DEBUG [StoreCloser-TestAcidGuarantees,,1731751162662.ffe1934c280093c181c2b062a05e0908.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/ffe1934c280093c181c2b062a05e0908/A/3160dd6fd84e4b82b402bd46540cc5ed to hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/archive/data/default/TestAcidGuarantees/ffe1934c280093c181c2b062a05e0908/A/3160dd6fd84e4b82b402bd46540cc5ed 2024-11-16T09:59:50,292 DEBUG [StoreCloser-TestAcidGuarantees,,1731751162662.ffe1934c280093c181c2b062a05e0908.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/ffe1934c280093c181c2b062a05e0908/A/496b53efeb1648a1ae19d0a8097b6c2b to hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/archive/data/default/TestAcidGuarantees/ffe1934c280093c181c2b062a05e0908/A/496b53efeb1648a1ae19d0a8097b6c2b 2024-11-16T09:59:50,293 DEBUG [StoreCloser-TestAcidGuarantees,,1731751162662.ffe1934c280093c181c2b062a05e0908.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/ffe1934c280093c181c2b062a05e0908/A/8e8ef152dca44dd8a3f29446df0b9790 to hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/archive/data/default/TestAcidGuarantees/ffe1934c280093c181c2b062a05e0908/A/8e8ef152dca44dd8a3f29446df0b9790 2024-11-16T09:59:50,294 DEBUG [StoreCloser-TestAcidGuarantees,,1731751162662.ffe1934c280093c181c2b062a05e0908.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/ffe1934c280093c181c2b062a05e0908/A/9e8350a74f8a4165b66b073a641f4577 to hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/archive/data/default/TestAcidGuarantees/ffe1934c280093c181c2b062a05e0908/A/9e8350a74f8a4165b66b073a641f4577 2024-11-16T09:59:50,295 DEBUG [StoreCloser-TestAcidGuarantees,,1731751162662.ffe1934c280093c181c2b062a05e0908.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/ffe1934c280093c181c2b062a05e0908/A/2bbc77228e7446d4826467c6853a1981 to hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/archive/data/default/TestAcidGuarantees/ffe1934c280093c181c2b062a05e0908/A/2bbc77228e7446d4826467c6853a1981 2024-11-16T09:59:50,295 DEBUG [StoreCloser-TestAcidGuarantees,,1731751162662.ffe1934c280093c181c2b062a05e0908.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/ffe1934c280093c181c2b062a05e0908/A/dac023d72c9a467cb301eb07f30695ca to hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/archive/data/default/TestAcidGuarantees/ffe1934c280093c181c2b062a05e0908/A/dac023d72c9a467cb301eb07f30695ca 2024-11-16T09:59:50,296 DEBUG [StoreCloser-TestAcidGuarantees,,1731751162662.ffe1934c280093c181c2b062a05e0908.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/ffe1934c280093c181c2b062a05e0908/A/e0361185f7ea491a8b4f44995bc8f102 to hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/archive/data/default/TestAcidGuarantees/ffe1934c280093c181c2b062a05e0908/A/e0361185f7ea491a8b4f44995bc8f102 2024-11-16T09:59:50,297 DEBUG [StoreCloser-TestAcidGuarantees,,1731751162662.ffe1934c280093c181c2b062a05e0908.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/ffe1934c280093c181c2b062a05e0908/A/8f7d13fd125a4e48ba6db638d37dd1d3 to hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/archive/data/default/TestAcidGuarantees/ffe1934c280093c181c2b062a05e0908/A/8f7d13fd125a4e48ba6db638d37dd1d3 2024-11-16T09:59:50,298 DEBUG [StoreCloser-TestAcidGuarantees,,1731751162662.ffe1934c280093c181c2b062a05e0908.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/ffe1934c280093c181c2b062a05e0908/A/fedbe5ffd79d430bbbbbeb61c1fe203e to hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/archive/data/default/TestAcidGuarantees/ffe1934c280093c181c2b062a05e0908/A/fedbe5ffd79d430bbbbbeb61c1fe203e 2024-11-16T09:59:50,299 DEBUG [StoreCloser-TestAcidGuarantees,,1731751162662.ffe1934c280093c181c2b062a05e0908.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/ffe1934c280093c181c2b062a05e0908/A/24bec8bf9c1c40a5afbc4caca203cc56 to hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/archive/data/default/TestAcidGuarantees/ffe1934c280093c181c2b062a05e0908/A/24bec8bf9c1c40a5afbc4caca203cc56 2024-11-16T09:59:50,300 DEBUG [StoreCloser-TestAcidGuarantees,,1731751162662.ffe1934c280093c181c2b062a05e0908.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/ffe1934c280093c181c2b062a05e0908/B/38d966a9b1fa4f7684b985da5492c745, hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/ffe1934c280093c181c2b062a05e0908/B/40ea47dbea964de1a868077b725e0626, hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/ffe1934c280093c181c2b062a05e0908/B/be4ceea2e8b74c3fb5af6601e8cc803b, hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/ffe1934c280093c181c2b062a05e0908/B/f0e157f5b6574298a053aade1bdb17e8, hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/ffe1934c280093c181c2b062a05e0908/B/d6f737504c674d9397fba019aa8885a5, hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/ffe1934c280093c181c2b062a05e0908/B/80fb85a1b63c419f8a6bc5905300e5f5, hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/ffe1934c280093c181c2b062a05e0908/B/02659887633940278476ed746fc4b512, hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/ffe1934c280093c181c2b062a05e0908/B/454a223dd32245fda138a60a07d8ee12, hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/ffe1934c280093c181c2b062a05e0908/B/8e4d5c1cfc8548b5b36ca5950cd01fc1, hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/ffe1934c280093c181c2b062a05e0908/B/3b3507fbba454d93a149ba91b104a66e, hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/ffe1934c280093c181c2b062a05e0908/B/50a26aec9b7c4e7dbbeffe873e11106f, hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/ffe1934c280093c181c2b062a05e0908/B/ea96bd3b9778485a8591866303bed84e, hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/ffe1934c280093c181c2b062a05e0908/B/32d1270366ac4a4688c1aae590fea7c5, hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/ffe1934c280093c181c2b062a05e0908/B/7ae7f0d8dc8547829a327b15ba583cf5, hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/ffe1934c280093c181c2b062a05e0908/B/fa2fc42df2b14c9788de3b052407407a, hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/ffe1934c280093c181c2b062a05e0908/B/04b0e51d058b49c7ac39bd0526239b72, hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/ffe1934c280093c181c2b062a05e0908/B/d66400cdff5b4ff79a783a59ef3e2994, hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/ffe1934c280093c181c2b062a05e0908/B/dc55e610407e49158964b1ae68c11f86, hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/ffe1934c280093c181c2b062a05e0908/B/2bd7c66293374d64944f326ac038a69b, hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/ffe1934c280093c181c2b062a05e0908/B/d208b33091da4873bc4b884835d2e72f, hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/ffe1934c280093c181c2b062a05e0908/B/a6e336fc389343dab5c447e05135b43f, hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/ffe1934c280093c181c2b062a05e0908/B/97d1a274f611400ab3a9315e75ebea67] to archive 2024-11-16T09:59:50,301 DEBUG [StoreCloser-TestAcidGuarantees,,1731751162662.ffe1934c280093c181c2b062a05e0908.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-16T09:59:50,302 DEBUG [StoreCloser-TestAcidGuarantees,,1731751162662.ffe1934c280093c181c2b062a05e0908.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/ffe1934c280093c181c2b062a05e0908/B/38d966a9b1fa4f7684b985da5492c745 to hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/archive/data/default/TestAcidGuarantees/ffe1934c280093c181c2b062a05e0908/B/38d966a9b1fa4f7684b985da5492c745 2024-11-16T09:59:50,303 DEBUG [StoreCloser-TestAcidGuarantees,,1731751162662.ffe1934c280093c181c2b062a05e0908.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/ffe1934c280093c181c2b062a05e0908/B/40ea47dbea964de1a868077b725e0626 to hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/archive/data/default/TestAcidGuarantees/ffe1934c280093c181c2b062a05e0908/B/40ea47dbea964de1a868077b725e0626 2024-11-16T09:59:50,303 DEBUG [StoreCloser-TestAcidGuarantees,,1731751162662.ffe1934c280093c181c2b062a05e0908.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/ffe1934c280093c181c2b062a05e0908/B/be4ceea2e8b74c3fb5af6601e8cc803b to hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/archive/data/default/TestAcidGuarantees/ffe1934c280093c181c2b062a05e0908/B/be4ceea2e8b74c3fb5af6601e8cc803b 2024-11-16T09:59:50,304 DEBUG [StoreCloser-TestAcidGuarantees,,1731751162662.ffe1934c280093c181c2b062a05e0908.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/ffe1934c280093c181c2b062a05e0908/B/f0e157f5b6574298a053aade1bdb17e8 to hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/archive/data/default/TestAcidGuarantees/ffe1934c280093c181c2b062a05e0908/B/f0e157f5b6574298a053aade1bdb17e8 2024-11-16T09:59:50,305 DEBUG [StoreCloser-TestAcidGuarantees,,1731751162662.ffe1934c280093c181c2b062a05e0908.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/ffe1934c280093c181c2b062a05e0908/B/d6f737504c674d9397fba019aa8885a5 to hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/archive/data/default/TestAcidGuarantees/ffe1934c280093c181c2b062a05e0908/B/d6f737504c674d9397fba019aa8885a5 2024-11-16T09:59:50,306 DEBUG [StoreCloser-TestAcidGuarantees,,1731751162662.ffe1934c280093c181c2b062a05e0908.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/ffe1934c280093c181c2b062a05e0908/B/80fb85a1b63c419f8a6bc5905300e5f5 to hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/archive/data/default/TestAcidGuarantees/ffe1934c280093c181c2b062a05e0908/B/80fb85a1b63c419f8a6bc5905300e5f5 2024-11-16T09:59:50,306 DEBUG [StoreCloser-TestAcidGuarantees,,1731751162662.ffe1934c280093c181c2b062a05e0908.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/ffe1934c280093c181c2b062a05e0908/B/02659887633940278476ed746fc4b512 to hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/archive/data/default/TestAcidGuarantees/ffe1934c280093c181c2b062a05e0908/B/02659887633940278476ed746fc4b512 2024-11-16T09:59:50,307 DEBUG [StoreCloser-TestAcidGuarantees,,1731751162662.ffe1934c280093c181c2b062a05e0908.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/ffe1934c280093c181c2b062a05e0908/B/454a223dd32245fda138a60a07d8ee12 to hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/archive/data/default/TestAcidGuarantees/ffe1934c280093c181c2b062a05e0908/B/454a223dd32245fda138a60a07d8ee12 2024-11-16T09:59:50,308 DEBUG [StoreCloser-TestAcidGuarantees,,1731751162662.ffe1934c280093c181c2b062a05e0908.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/ffe1934c280093c181c2b062a05e0908/B/8e4d5c1cfc8548b5b36ca5950cd01fc1 to hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/archive/data/default/TestAcidGuarantees/ffe1934c280093c181c2b062a05e0908/B/8e4d5c1cfc8548b5b36ca5950cd01fc1 2024-11-16T09:59:50,309 DEBUG [StoreCloser-TestAcidGuarantees,,1731751162662.ffe1934c280093c181c2b062a05e0908.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/ffe1934c280093c181c2b062a05e0908/B/3b3507fbba454d93a149ba91b104a66e to hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/archive/data/default/TestAcidGuarantees/ffe1934c280093c181c2b062a05e0908/B/3b3507fbba454d93a149ba91b104a66e 2024-11-16T09:59:50,309 DEBUG [StoreCloser-TestAcidGuarantees,,1731751162662.ffe1934c280093c181c2b062a05e0908.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/ffe1934c280093c181c2b062a05e0908/B/50a26aec9b7c4e7dbbeffe873e11106f to hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/archive/data/default/TestAcidGuarantees/ffe1934c280093c181c2b062a05e0908/B/50a26aec9b7c4e7dbbeffe873e11106f 2024-11-16T09:59:50,310 DEBUG [StoreCloser-TestAcidGuarantees,,1731751162662.ffe1934c280093c181c2b062a05e0908.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/ffe1934c280093c181c2b062a05e0908/B/ea96bd3b9778485a8591866303bed84e to hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/archive/data/default/TestAcidGuarantees/ffe1934c280093c181c2b062a05e0908/B/ea96bd3b9778485a8591866303bed84e 2024-11-16T09:59:50,311 DEBUG [StoreCloser-TestAcidGuarantees,,1731751162662.ffe1934c280093c181c2b062a05e0908.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/ffe1934c280093c181c2b062a05e0908/B/32d1270366ac4a4688c1aae590fea7c5 to hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/archive/data/default/TestAcidGuarantees/ffe1934c280093c181c2b062a05e0908/B/32d1270366ac4a4688c1aae590fea7c5 2024-11-16T09:59:50,312 DEBUG [StoreCloser-TestAcidGuarantees,,1731751162662.ffe1934c280093c181c2b062a05e0908.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/ffe1934c280093c181c2b062a05e0908/B/7ae7f0d8dc8547829a327b15ba583cf5 to hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/archive/data/default/TestAcidGuarantees/ffe1934c280093c181c2b062a05e0908/B/7ae7f0d8dc8547829a327b15ba583cf5 2024-11-16T09:59:50,313 DEBUG [StoreCloser-TestAcidGuarantees,,1731751162662.ffe1934c280093c181c2b062a05e0908.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/ffe1934c280093c181c2b062a05e0908/B/fa2fc42df2b14c9788de3b052407407a to hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/archive/data/default/TestAcidGuarantees/ffe1934c280093c181c2b062a05e0908/B/fa2fc42df2b14c9788de3b052407407a 2024-11-16T09:59:50,314 DEBUG [StoreCloser-TestAcidGuarantees,,1731751162662.ffe1934c280093c181c2b062a05e0908.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/ffe1934c280093c181c2b062a05e0908/B/04b0e51d058b49c7ac39bd0526239b72 to hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/archive/data/default/TestAcidGuarantees/ffe1934c280093c181c2b062a05e0908/B/04b0e51d058b49c7ac39bd0526239b72 2024-11-16T09:59:50,315 DEBUG [StoreCloser-TestAcidGuarantees,,1731751162662.ffe1934c280093c181c2b062a05e0908.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/ffe1934c280093c181c2b062a05e0908/B/d66400cdff5b4ff79a783a59ef3e2994 to hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/archive/data/default/TestAcidGuarantees/ffe1934c280093c181c2b062a05e0908/B/d66400cdff5b4ff79a783a59ef3e2994 2024-11-16T09:59:50,316 DEBUG [StoreCloser-TestAcidGuarantees,,1731751162662.ffe1934c280093c181c2b062a05e0908.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/ffe1934c280093c181c2b062a05e0908/B/dc55e610407e49158964b1ae68c11f86 to hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/archive/data/default/TestAcidGuarantees/ffe1934c280093c181c2b062a05e0908/B/dc55e610407e49158964b1ae68c11f86 2024-11-16T09:59:50,316 DEBUG [StoreCloser-TestAcidGuarantees,,1731751162662.ffe1934c280093c181c2b062a05e0908.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/ffe1934c280093c181c2b062a05e0908/B/2bd7c66293374d64944f326ac038a69b to hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/archive/data/default/TestAcidGuarantees/ffe1934c280093c181c2b062a05e0908/B/2bd7c66293374d64944f326ac038a69b 2024-11-16T09:59:50,317 DEBUG [StoreCloser-TestAcidGuarantees,,1731751162662.ffe1934c280093c181c2b062a05e0908.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/ffe1934c280093c181c2b062a05e0908/B/d208b33091da4873bc4b884835d2e72f to hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/archive/data/default/TestAcidGuarantees/ffe1934c280093c181c2b062a05e0908/B/d208b33091da4873bc4b884835d2e72f 2024-11-16T09:59:50,318 DEBUG [StoreCloser-TestAcidGuarantees,,1731751162662.ffe1934c280093c181c2b062a05e0908.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/ffe1934c280093c181c2b062a05e0908/B/a6e336fc389343dab5c447e05135b43f to hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/archive/data/default/TestAcidGuarantees/ffe1934c280093c181c2b062a05e0908/B/a6e336fc389343dab5c447e05135b43f 2024-11-16T09:59:50,319 DEBUG [StoreCloser-TestAcidGuarantees,,1731751162662.ffe1934c280093c181c2b062a05e0908.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/ffe1934c280093c181c2b062a05e0908/B/97d1a274f611400ab3a9315e75ebea67 to hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/archive/data/default/TestAcidGuarantees/ffe1934c280093c181c2b062a05e0908/B/97d1a274f611400ab3a9315e75ebea67 2024-11-16T09:59:50,321 DEBUG [StoreCloser-TestAcidGuarantees,,1731751162662.ffe1934c280093c181c2b062a05e0908.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/ffe1934c280093c181c2b062a05e0908/C/5d8b2f5d76744a4a935a06e9f2b023ea, hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/ffe1934c280093c181c2b062a05e0908/C/9ad2541e56924229b4932313705f9787, hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/ffe1934c280093c181c2b062a05e0908/C/ca4c37e2417b4a24b1ff87c6e2fd47f9, hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/ffe1934c280093c181c2b062a05e0908/C/7fa927d3441046e0962c03ddbd3858b7, hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/ffe1934c280093c181c2b062a05e0908/C/42861dc2507c4c0f8d9b4b330a334f9b, hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/ffe1934c280093c181c2b062a05e0908/C/411f65ffbe6f434caf13db4742086cea, hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/ffe1934c280093c181c2b062a05e0908/C/a53a9c0af5e648b2bedbfa0d8cb8a65b, hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/ffe1934c280093c181c2b062a05e0908/C/c5a878457eff483bb100e1b51d494e7f, hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/ffe1934c280093c181c2b062a05e0908/C/080481f61e514e57b5256885a8981750, hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/ffe1934c280093c181c2b062a05e0908/C/ed38a46f975e446c9c74ecd764de8df0, hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/ffe1934c280093c181c2b062a05e0908/C/627b560ebad44444b790f9f1742da379, hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/ffe1934c280093c181c2b062a05e0908/C/6d07f7417b00489c94662f29698743f1, hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/ffe1934c280093c181c2b062a05e0908/C/5ec075b689da412bb397e9b9dbafdfe3, hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/ffe1934c280093c181c2b062a05e0908/C/20f06da3d4ca4b91a578717bd9dbaee0, hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/ffe1934c280093c181c2b062a05e0908/C/a0afb7c2bda64ca680e563efa5618949, hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/ffe1934c280093c181c2b062a05e0908/C/c498d0fe177440029170dc63c55c9c94, hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/ffe1934c280093c181c2b062a05e0908/C/715709ecd09c44af93fb2375ba1dbdbf, hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/ffe1934c280093c181c2b062a05e0908/C/0f4fae61ac1e4e00bebc2ddf2341d072, hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/ffe1934c280093c181c2b062a05e0908/C/66cc8e420ade4f18ab524bd130018691] to archive 2024-11-16T09:59:50,321 DEBUG [StoreCloser-TestAcidGuarantees,,1731751162662.ffe1934c280093c181c2b062a05e0908.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-16T09:59:50,322 DEBUG [StoreCloser-TestAcidGuarantees,,1731751162662.ffe1934c280093c181c2b062a05e0908.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/ffe1934c280093c181c2b062a05e0908/C/5d8b2f5d76744a4a935a06e9f2b023ea to hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/archive/data/default/TestAcidGuarantees/ffe1934c280093c181c2b062a05e0908/C/5d8b2f5d76744a4a935a06e9f2b023ea 2024-11-16T09:59:50,323 DEBUG [StoreCloser-TestAcidGuarantees,,1731751162662.ffe1934c280093c181c2b062a05e0908.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/ffe1934c280093c181c2b062a05e0908/C/9ad2541e56924229b4932313705f9787 to hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/archive/data/default/TestAcidGuarantees/ffe1934c280093c181c2b062a05e0908/C/9ad2541e56924229b4932313705f9787 2024-11-16T09:59:50,324 DEBUG [StoreCloser-TestAcidGuarantees,,1731751162662.ffe1934c280093c181c2b062a05e0908.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/ffe1934c280093c181c2b062a05e0908/C/ca4c37e2417b4a24b1ff87c6e2fd47f9 to hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/archive/data/default/TestAcidGuarantees/ffe1934c280093c181c2b062a05e0908/C/ca4c37e2417b4a24b1ff87c6e2fd47f9 2024-11-16T09:59:50,324 DEBUG [StoreCloser-TestAcidGuarantees,,1731751162662.ffe1934c280093c181c2b062a05e0908.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/ffe1934c280093c181c2b062a05e0908/C/7fa927d3441046e0962c03ddbd3858b7 to hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/archive/data/default/TestAcidGuarantees/ffe1934c280093c181c2b062a05e0908/C/7fa927d3441046e0962c03ddbd3858b7 2024-11-16T09:59:50,325 DEBUG [StoreCloser-TestAcidGuarantees,,1731751162662.ffe1934c280093c181c2b062a05e0908.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/ffe1934c280093c181c2b062a05e0908/C/42861dc2507c4c0f8d9b4b330a334f9b to hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/archive/data/default/TestAcidGuarantees/ffe1934c280093c181c2b062a05e0908/C/42861dc2507c4c0f8d9b4b330a334f9b 2024-11-16T09:59:50,326 DEBUG [StoreCloser-TestAcidGuarantees,,1731751162662.ffe1934c280093c181c2b062a05e0908.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/ffe1934c280093c181c2b062a05e0908/C/411f65ffbe6f434caf13db4742086cea to hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/archive/data/default/TestAcidGuarantees/ffe1934c280093c181c2b062a05e0908/C/411f65ffbe6f434caf13db4742086cea 2024-11-16T09:59:50,326 DEBUG [StoreCloser-TestAcidGuarantees,,1731751162662.ffe1934c280093c181c2b062a05e0908.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/ffe1934c280093c181c2b062a05e0908/C/a53a9c0af5e648b2bedbfa0d8cb8a65b to hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/archive/data/default/TestAcidGuarantees/ffe1934c280093c181c2b062a05e0908/C/a53a9c0af5e648b2bedbfa0d8cb8a65b 2024-11-16T09:59:50,327 DEBUG [StoreCloser-TestAcidGuarantees,,1731751162662.ffe1934c280093c181c2b062a05e0908.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/ffe1934c280093c181c2b062a05e0908/C/c5a878457eff483bb100e1b51d494e7f to hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/archive/data/default/TestAcidGuarantees/ffe1934c280093c181c2b062a05e0908/C/c5a878457eff483bb100e1b51d494e7f 2024-11-16T09:59:50,328 DEBUG [StoreCloser-TestAcidGuarantees,,1731751162662.ffe1934c280093c181c2b062a05e0908.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/ffe1934c280093c181c2b062a05e0908/C/080481f61e514e57b5256885a8981750 to hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/archive/data/default/TestAcidGuarantees/ffe1934c280093c181c2b062a05e0908/C/080481f61e514e57b5256885a8981750 2024-11-16T09:59:50,329 DEBUG [StoreCloser-TestAcidGuarantees,,1731751162662.ffe1934c280093c181c2b062a05e0908.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/ffe1934c280093c181c2b062a05e0908/C/ed38a46f975e446c9c74ecd764de8df0 to hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/archive/data/default/TestAcidGuarantees/ffe1934c280093c181c2b062a05e0908/C/ed38a46f975e446c9c74ecd764de8df0 2024-11-16T09:59:50,329 DEBUG [StoreCloser-TestAcidGuarantees,,1731751162662.ffe1934c280093c181c2b062a05e0908.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/ffe1934c280093c181c2b062a05e0908/C/627b560ebad44444b790f9f1742da379 to hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/archive/data/default/TestAcidGuarantees/ffe1934c280093c181c2b062a05e0908/C/627b560ebad44444b790f9f1742da379 2024-11-16T09:59:50,330 DEBUG [StoreCloser-TestAcidGuarantees,,1731751162662.ffe1934c280093c181c2b062a05e0908.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/ffe1934c280093c181c2b062a05e0908/C/6d07f7417b00489c94662f29698743f1 to hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/archive/data/default/TestAcidGuarantees/ffe1934c280093c181c2b062a05e0908/C/6d07f7417b00489c94662f29698743f1 2024-11-16T09:59:50,331 DEBUG [StoreCloser-TestAcidGuarantees,,1731751162662.ffe1934c280093c181c2b062a05e0908.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/ffe1934c280093c181c2b062a05e0908/C/5ec075b689da412bb397e9b9dbafdfe3 to hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/archive/data/default/TestAcidGuarantees/ffe1934c280093c181c2b062a05e0908/C/5ec075b689da412bb397e9b9dbafdfe3 2024-11-16T09:59:50,332 DEBUG [StoreCloser-TestAcidGuarantees,,1731751162662.ffe1934c280093c181c2b062a05e0908.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/ffe1934c280093c181c2b062a05e0908/C/20f06da3d4ca4b91a578717bd9dbaee0 to hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/archive/data/default/TestAcidGuarantees/ffe1934c280093c181c2b062a05e0908/C/20f06da3d4ca4b91a578717bd9dbaee0 2024-11-16T09:59:50,332 DEBUG [StoreCloser-TestAcidGuarantees,,1731751162662.ffe1934c280093c181c2b062a05e0908.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/ffe1934c280093c181c2b062a05e0908/C/a0afb7c2bda64ca680e563efa5618949 to hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/archive/data/default/TestAcidGuarantees/ffe1934c280093c181c2b062a05e0908/C/a0afb7c2bda64ca680e563efa5618949 2024-11-16T09:59:50,333 DEBUG [StoreCloser-TestAcidGuarantees,,1731751162662.ffe1934c280093c181c2b062a05e0908.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/ffe1934c280093c181c2b062a05e0908/C/c498d0fe177440029170dc63c55c9c94 to hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/archive/data/default/TestAcidGuarantees/ffe1934c280093c181c2b062a05e0908/C/c498d0fe177440029170dc63c55c9c94 2024-11-16T09:59:50,334 DEBUG [StoreCloser-TestAcidGuarantees,,1731751162662.ffe1934c280093c181c2b062a05e0908.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/ffe1934c280093c181c2b062a05e0908/C/715709ecd09c44af93fb2375ba1dbdbf to hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/archive/data/default/TestAcidGuarantees/ffe1934c280093c181c2b062a05e0908/C/715709ecd09c44af93fb2375ba1dbdbf 2024-11-16T09:59:50,335 DEBUG [StoreCloser-TestAcidGuarantees,,1731751162662.ffe1934c280093c181c2b062a05e0908.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/ffe1934c280093c181c2b062a05e0908/C/0f4fae61ac1e4e00bebc2ddf2341d072 to hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/archive/data/default/TestAcidGuarantees/ffe1934c280093c181c2b062a05e0908/C/0f4fae61ac1e4e00bebc2ddf2341d072 2024-11-16T09:59:50,336 DEBUG [StoreCloser-TestAcidGuarantees,,1731751162662.ffe1934c280093c181c2b062a05e0908.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/ffe1934c280093c181c2b062a05e0908/C/66cc8e420ade4f18ab524bd130018691 to hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/archive/data/default/TestAcidGuarantees/ffe1934c280093c181c2b062a05e0908/C/66cc8e420ade4f18ab524bd130018691 2024-11-16T09:59:50,339 DEBUG [RS_CLOSE_REGION-regionserver/037c58a19ddb:0-0 {event_type=M_RS_CLOSE_REGION, pid=114}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/ffe1934c280093c181c2b062a05e0908/recovered.edits/319.seqid, newMaxSeqId=319, maxSeqId=4 2024-11-16T09:59:50,340 INFO [RS_CLOSE_REGION-regionserver/037c58a19ddb:0-0 {event_type=M_RS_CLOSE_REGION, pid=114}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1731751162662.ffe1934c280093c181c2b062a05e0908. 2024-11-16T09:59:50,340 DEBUG [RS_CLOSE_REGION-regionserver/037c58a19ddb:0-0 {event_type=M_RS_CLOSE_REGION, pid=114}] regionserver.HRegion(1676): Region close journal for ffe1934c280093c181c2b062a05e0908: Waiting for close lock at 1731751188050Running coprocessor pre-close hooks at 1731751188050Disabling compacts and flushes for region at 1731751188050Disabling writes for close at 1731751188630 (+580 ms)Obtaining lock to block concurrent updates at 1731751188630Preparing flush snapshotting stores in ffe1934c280093c181c2b062a05e0908 at 1731751188630Finished memstore snapshotting TestAcidGuarantees,,1731751162662.ffe1934c280093c181c2b062a05e0908., syncing WAL and waiting on mvcc, flushsize=dataSize=6870, getHeapSize=18720, getOffHeapSize=0, getCellsCount=150 at 1731751188630Flushing stores of TestAcidGuarantees,,1731751162662.ffe1934c280093c181c2b062a05e0908. at 1731751188630Flushing ffe1934c280093c181c2b062a05e0908/A: creating writer at 1731751188630Flushing mob file ffe1934c280093c181c2b062a05e0908/A: appending metadata at 1731751188634 (+4 ms)Flushing mob file ffe1934c280093c181c2b062a05e0908/A: closing flushed file at 1731751188634Flushing ffe1934c280093c181c2b062a05e0908/A: appending metadata at 1731751189042 (+408 ms)Flushing ffe1934c280093c181c2b062a05e0908/A: closing flushed file at 1731751189042Flushing ffe1934c280093c181c2b062a05e0908/B: creating writer at 1731751189450 (+408 ms)Flushing ffe1934c280093c181c2b062a05e0908/B: appending metadata at 1731751189452 (+2 ms)Flushing ffe1934c280093c181c2b062a05e0908/B: closing flushed file at 1731751189452Flushing ffe1934c280093c181c2b062a05e0908/C: creating writer at 1731751189859 (+407 ms)Flushing ffe1934c280093c181c2b062a05e0908/C: appending metadata at 1731751189862 (+3 ms)Flushing ffe1934c280093c181c2b062a05e0908/C: closing flushed file at 1731751189862Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@1bb1e796: reopening flushed file at 1731751190269 (+407 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@432b5023: reopening flushed file at 1731751190272 (+3 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@750025c2: reopening flushed file at 1731751190276 (+4 ms)Finished flush of dataSize ~6.71 KB/6870, heapSize ~18.28 KB/18720, currentSize=0 B/0 for ffe1934c280093c181c2b062a05e0908 in 1649ms, sequenceid=316, compaction requested=true at 1731751190280 (+4 ms)Writing region close event to WAL at 1731751190337 (+57 ms)Running coprocessor post-close hooks at 1731751190340 (+3 ms)Closed at 1731751190340 2024-11-16T09:59:50,342 INFO [RS_CLOSE_REGION-regionserver/037c58a19ddb:0-0 {event_type=M_RS_CLOSE_REGION, pid=114}] handler.UnassignRegionHandler(157): Closed ffe1934c280093c181c2b062a05e0908 2024-11-16T09:59:50,342 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=113 updating hbase:meta row=ffe1934c280093c181c2b062a05e0908, regionState=CLOSED 2024-11-16T09:59:50,343 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=114, ppid=113, state=RUNNABLE, hasLock=false; CloseRegionProcedure ffe1934c280093c181c2b062a05e0908, server=037c58a19ddb,37267,1731751077844 because future has completed 2024-11-16T09:59:50,345 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=114, resume processing ppid=113 2024-11-16T09:59:50,345 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=114, ppid=113, state=SUCCESS, hasLock=false; CloseRegionProcedure ffe1934c280093c181c2b062a05e0908, server=037c58a19ddb,37267,1731751077844 in 2.4470 sec 2024-11-16T09:59:50,347 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=113, resume processing ppid=112 2024-11-16T09:59:50,347 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=113, ppid=112, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=ffe1934c280093c181c2b062a05e0908, UNASSIGN in 2.4510 sec 2024-11-16T09:59:50,348 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=112, resume processing ppid=111 2024-11-16T09:59:50,348 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=112, ppid=111, state=SUCCESS, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees in 2.4550 sec 2024-11-16T09:59:50,349 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731751190349"}]},"ts":"1731751190349"} 2024-11-16T09:59:50,350 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLED in hbase:meta 2024-11-16T09:59:50,350 INFO [PEWorker-4 {}] procedure.DisableTableProcedure(296): Set TestAcidGuarantees to state=DISABLED 2024-11-16T09:59:50,352 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=111, state=SUCCESS, hasLock=false; DisableTableProcedure table=TestAcidGuarantees in 2.4650 sec 2024-11-16T09:59:52,053 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37877 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=111 2024-11-16T09:59:52,053 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DISABLE, Table Name: default:TestAcidGuarantees completed 2024-11-16T09:59:52,054 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37877 {}] master.HMaster$5(2570): Client=jenkins//172.17.0.2 delete TestAcidGuarantees 2024-11-16T09:59:52,054 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37877 {}] procedure2.ProcedureExecutor(1139): Stored pid=115, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees 2024-11-16T09:59:52,055 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(103): Waiting for RIT for pid=115, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-16T09:59:52,056 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(115): Deleting regions from filesystem for pid=115, state=RUNNABLE:DELETE_TABLE_CLEAR_FS_LAYOUT, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-16T09:59:52,056 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37877 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=115 2024-11-16T09:59:52,059 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(131): ARCHIVING hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/ffe1934c280093c181c2b062a05e0908 2024-11-16T09:59:52,060 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(159): Archiving [FileablePath, hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/ffe1934c280093c181c2b062a05e0908/A, FileablePath, hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/ffe1934c280093c181c2b062a05e0908/B, FileablePath, hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/ffe1934c280093c181c2b062a05e0908/C, FileablePath, hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/ffe1934c280093c181c2b062a05e0908/recovered.edits] 2024-11-16T09:59:52,063 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/ffe1934c280093c181c2b062a05e0908/A/a8b81e40e710410394660fce77f9a304 to hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/archive/data/default/TestAcidGuarantees/ffe1934c280093c181c2b062a05e0908/A/a8b81e40e710410394660fce77f9a304 2024-11-16T09:59:52,063 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/ffe1934c280093c181c2b062a05e0908/A/fb3563ac534d4a618da4ec8b60e77b17 to hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/archive/data/default/TestAcidGuarantees/ffe1934c280093c181c2b062a05e0908/A/fb3563ac534d4a618da4ec8b60e77b17 2024-11-16T09:59:52,065 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/ffe1934c280093c181c2b062a05e0908/B/5f9e7a026fb94fa19c5eb278dead9713 to hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/archive/data/default/TestAcidGuarantees/ffe1934c280093c181c2b062a05e0908/B/5f9e7a026fb94fa19c5eb278dead9713 2024-11-16T09:59:52,066 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/ffe1934c280093c181c2b062a05e0908/B/739d447954a74addafe8c957d46f719f to hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/archive/data/default/TestAcidGuarantees/ffe1934c280093c181c2b062a05e0908/B/739d447954a74addafe8c957d46f719f 2024-11-16T09:59:52,068 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/ffe1934c280093c181c2b062a05e0908/C/1cbc6d0426af4c569b8949b7f82a3b73 to hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/archive/data/default/TestAcidGuarantees/ffe1934c280093c181c2b062a05e0908/C/1cbc6d0426af4c569b8949b7f82a3b73 2024-11-16T09:59:52,069 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/ffe1934c280093c181c2b062a05e0908/C/41feadaaa74748549c2f8cde91937b45 to hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/archive/data/default/TestAcidGuarantees/ffe1934c280093c181c2b062a05e0908/C/41feadaaa74748549c2f8cde91937b45 2024-11-16T09:59:52,070 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/ffe1934c280093c181c2b062a05e0908/C/92d4b326ce914ac591b817dc8a4ea75f to hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/archive/data/default/TestAcidGuarantees/ffe1934c280093c181c2b062a05e0908/C/92d4b326ce914ac591b817dc8a4ea75f 2024-11-16T09:59:52,070 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/ffe1934c280093c181c2b062a05e0908/C/a0dae046833e43a299803c1a8906fdcb to hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/archive/data/default/TestAcidGuarantees/ffe1934c280093c181c2b062a05e0908/C/a0dae046833e43a299803c1a8906fdcb 2024-11-16T09:59:52,072 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/ffe1934c280093c181c2b062a05e0908/recovered.edits/319.seqid to hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/archive/data/default/TestAcidGuarantees/ffe1934c280093c181c2b062a05e0908/recovered.edits/319.seqid 2024-11-16T09:59:52,073 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(610): Deleted hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/ffe1934c280093c181c2b062a05e0908 2024-11-16T09:59:52,073 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(313): Archived TestAcidGuarantees regions 2024-11-16T09:59:52,073 DEBUG [PEWorker-3 {}] backup.HFileArchiver(131): ARCHIVING hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3 2024-11-16T09:59:52,074 DEBUG [PEWorker-3 {}] backup.HFileArchiver(159): Archiving [FileablePath, hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A] 2024-11-16T09:59:52,076 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411160435bee7c2b743d3892878423fb632a0_ffe1934c280093c181c2b062a05e0908 to hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411160435bee7c2b743d3892878423fb632a0_ffe1934c280093c181c2b062a05e0908 2024-11-16T09:59:52,077 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411160b7c32d8d96e45b0812750bb24a9975c_ffe1934c280093c181c2b062a05e0908 to hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411160b7c32d8d96e45b0812750bb24a9975c_ffe1934c280093c181c2b062a05e0908 2024-11-16T09:59:52,078 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111611a0e4b311e040019efb82a03d8d338f_ffe1934c280093c181c2b062a05e0908 to hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111611a0e4b311e040019efb82a03d8d338f_ffe1934c280093c181c2b062a05e0908 2024-11-16T09:59:52,078 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411162584b4e9ec0542f2ae8795cee9ff949a_ffe1934c280093c181c2b062a05e0908 to hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411162584b4e9ec0542f2ae8795cee9ff949a_ffe1934c280093c181c2b062a05e0908 2024-11-16T09:59:52,079 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111656efabdd34124ec596e7ad480ca80c50_ffe1934c280093c181c2b062a05e0908 to hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111656efabdd34124ec596e7ad480ca80c50_ffe1934c280093c181c2b062a05e0908 2024-11-16T09:59:52,080 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411165d7175141f574c2c8081aa20a6cc4783_ffe1934c280093c181c2b062a05e0908 to hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411165d7175141f574c2c8081aa20a6cc4783_ffe1934c280093c181c2b062a05e0908 2024-11-16T09:59:52,081 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111660cb07e574154cb98add16287a603d40_ffe1934c280093c181c2b062a05e0908 to hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111660cb07e574154cb98add16287a603d40_ffe1934c280093c181c2b062a05e0908 2024-11-16T09:59:52,082 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411166857ff57a2f341f699ea264dea73761a_ffe1934c280093c181c2b062a05e0908 to hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411166857ff57a2f341f699ea264dea73761a_ffe1934c280093c181c2b062a05e0908 2024-11-16T09:59:52,082 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111676c0b879531f4f18a7ce5c7bdde3a78c_ffe1934c280093c181c2b062a05e0908 to hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111676c0b879531f4f18a7ce5c7bdde3a78c_ffe1934c280093c181c2b062a05e0908 2024-11-16T09:59:52,083 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411167b12dd1e49d4494db2d8453572b63c0f_ffe1934c280093c181c2b062a05e0908 to hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411167b12dd1e49d4494db2d8453572b63c0f_ffe1934c280093c181c2b062a05e0908 2024-11-16T09:59:52,084 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241116822785b22ac548a6831fc5b09e62d398_ffe1934c280093c181c2b062a05e0908 to hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241116822785b22ac548a6831fc5b09e62d398_ffe1934c280093c181c2b062a05e0908 2024-11-16T09:59:52,085 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411169aa6866b5eee4abdaec697d6914e75ce_ffe1934c280093c181c2b062a05e0908 to hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411169aa6866b5eee4abdaec697d6914e75ce_ffe1934c280093c181c2b062a05e0908 2024-11-16T09:59:52,086 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241116beb0e73d2beb4d2f872976e5b9933168_ffe1934c280093c181c2b062a05e0908 to hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241116beb0e73d2beb4d2f872976e5b9933168_ffe1934c280093c181c2b062a05e0908 2024-11-16T09:59:52,087 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241116d2ccce757c644c5aac4dbb8dce2a1858_ffe1934c280093c181c2b062a05e0908 to hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241116d2ccce757c644c5aac4dbb8dce2a1858_ffe1934c280093c181c2b062a05e0908 2024-11-16T09:59:52,088 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241116d8158e3a222d4016baa85be596ba0e39_ffe1934c280093c181c2b062a05e0908 to hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241116d8158e3a222d4016baa85be596ba0e39_ffe1934c280093c181c2b062a05e0908 2024-11-16T09:59:52,089 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241116f2e8ccce4d6b45ff8661f0b31a5e0b62_ffe1934c280093c181c2b062a05e0908 to hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241116f2e8ccce4d6b45ff8661f0b31a5e0b62_ffe1934c280093c181c2b062a05e0908 2024-11-16T09:59:52,090 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241116fc79cb79698c4c83bc45db23fbe52c09_ffe1934c280093c181c2b062a05e0908 to hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241116fc79cb79698c4c83bc45db23fbe52c09_ffe1934c280093c181c2b062a05e0908 2024-11-16T09:59:52,090 DEBUG [PEWorker-3 {}] backup.HFileArchiver(610): Deleted hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3 2024-11-16T09:59:52,092 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(120): Deleting regions from META for pid=115, state=RUNNABLE:DELETE_TABLE_REMOVE_FROM_META, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-16T09:59:52,094 WARN [PEWorker-3 {}] procedure.DeleteTableProcedure(370): Deleting some vestigial 1 rows of TestAcidGuarantees from hbase:meta 2024-11-16T09:59:52,096 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(407): Removing 'TestAcidGuarantees' descriptor. 2024-11-16T09:59:52,097 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(126): Deleting assignment state for pid=115, state=RUNNABLE:DELETE_TABLE_UNASSIGN_REGIONS, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-16T09:59:52,097 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(397): Removing 'TestAcidGuarantees' from region states. 2024-11-16T09:59:52,098 DEBUG [PEWorker-3 {}] assignment.RegionStateStore(723): Delete {"totalColumns":1,"row":"TestAcidGuarantees,,1731751162662.ffe1934c280093c181c2b062a05e0908.","families":{"info":[{"qualifier":"","vlen":0,"tag":[],"timestamp":"1731751192097"}]},"ts":"9223372036854775807"} 2024-11-16T09:59:52,099 INFO [PEWorker-3 {}] assignment.RegionStateStore(562): Deleted 1 regions from META 2024-11-16T09:59:52,099 DEBUG [PEWorker-3 {}] assignment.RegionStateStore(563): Deleted regions: [{ENCODED => ffe1934c280093c181c2b062a05e0908, NAME => 'TestAcidGuarantees,,1731751162662.ffe1934c280093c181c2b062a05e0908.', STARTKEY => '', ENDKEY => ''}] 2024-11-16T09:59:52,099 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(401): Marking 'TestAcidGuarantees' as deleted. 2024-11-16T09:59:52,100 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Delete {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":0,"tag":[],"timestamp":"1731751192099"}]},"ts":"9223372036854775807"} 2024-11-16T09:59:52,101 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(867): Deleted table TestAcidGuarantees state from META 2024-11-16T09:59:52,102 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(133): Finished pid=115, state=RUNNABLE:DELETE_TABLE_POST_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-16T09:59:52,103 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=115, state=SUCCESS, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees in 48 msec 2024-11-16T09:59:52,163 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37877 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=115 2024-11-16T09:59:52,163 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocator(219): Clear meta cache for TestAcidGuarantees 2024-11-16T09:59:52,163 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DELETE, Table Name: default:TestAcidGuarantees completed 2024-11-16T09:59:52,172 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: TestAcidGuaranteesWithNoInMemCompaction#testMobScanAtomicity Thread=242 (was 241) - Thread LEAK? -, OpenFileDescriptor=459 (was 457) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=304 (was 361), ProcessCount=11 (was 11), AvailableMemoryMB=5850 (was 5995) 2024-11-16T09:59:52,180 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: TestAcidGuaranteesWithNoInMemCompaction#testScanAtomicity Thread=242, OpenFileDescriptor=459, MaxFileDescriptor=1048576, SystemLoadAverage=304, ProcessCount=11, AvailableMemoryMB=5849 2024-11-16T09:59:52,181 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37877 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (131072) is too small, which might cause very frequent flushing. 2024-11-16T09:59:52,181 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37877 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.2 create 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'NONE'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-16T09:59:52,182 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37877 {}] procedure2.ProcedureExecutor(1139): Stored pid=116, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=TestAcidGuarantees 2024-11-16T09:59:52,183 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=116, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_PRE_OPERATION 2024-11-16T09:59:52,183 DEBUG [PEWorker-2 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T09:59:52,183 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37877 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.2 procedure request for creating table: namespace: "default" qualifier: "TestAcidGuarantees" procId is: 116 2024-11-16T09:59:52,184 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37877 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=116 2024-11-16T09:59:52,184 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=116, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-11-16T09:59:52,188 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37591 is added to blk_1073742266_1442 (size=959) 2024-11-16T09:59:52,293 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37877 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=116 2024-11-16T09:59:52,504 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37877 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=116 2024-11-16T09:59:52,590 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => 697cd6a363e514bff0d4ec954fc4034b, NAME => 'TestAcidGuarantees,,1731751192181.697cd6a363e514bff0d4ec954fc4034b.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'NONE', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592 2024-11-16T09:59:52,595 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37591 is added to blk_1073742267_1443 (size=53) 2024-11-16T09:59:52,813 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37877 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=116 2024-11-16T09:59:52,996 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1731751192181.697cd6a363e514bff0d4ec954fc4034b.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-16T09:59:52,996 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1722): Closing 697cd6a363e514bff0d4ec954fc4034b, disabling compactions & flushes 2024-11-16T09:59:52,996 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1731751192181.697cd6a363e514bff0d4ec954fc4034b. 2024-11-16T09:59:52,996 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1731751192181.697cd6a363e514bff0d4ec954fc4034b. 2024-11-16T09:59:52,996 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1731751192181.697cd6a363e514bff0d4ec954fc4034b. after waiting 0 ms 2024-11-16T09:59:52,996 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1731751192181.697cd6a363e514bff0d4ec954fc4034b. 2024-11-16T09:59:52,996 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1731751192181.697cd6a363e514bff0d4ec954fc4034b. 2024-11-16T09:59:52,996 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1676): Region close journal for 697cd6a363e514bff0d4ec954fc4034b: Waiting for close lock at 1731751192996Disabling compacts and flushes for region at 1731751192996Disabling writes for close at 1731751192996Writing region close event to WAL at 1731751192996Closed at 1731751192996 2024-11-16T09:59:52,997 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=116, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ADD_TO_META 2024-11-16T09:59:52,997 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"TestAcidGuarantees,,1731751192181.697cd6a363e514bff0d4ec954fc4034b.","families":{"info":[{"qualifier":"regioninfo","vlen":52,"tag":[],"timestamp":"1731751192997"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1731751192997"}]},"ts":"1731751192997"} 2024-11-16T09:59:52,999 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-11-16T09:59:52,999 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=116, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-11-16T09:59:52,999 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731751192999"}]},"ts":"1731751192999"} 2024-11-16T09:59:53,001 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLING in hbase:meta 2024-11-16T09:59:53,001 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=117, ppid=116, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=697cd6a363e514bff0d4ec954fc4034b, ASSIGN}] 2024-11-16T09:59:53,002 INFO [PEWorker-5 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=117, ppid=116, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=697cd6a363e514bff0d4ec954fc4034b, ASSIGN 2024-11-16T09:59:53,002 INFO [PEWorker-5 {}] assignment.TransitRegionStateProcedure(269): Starting pid=117, ppid=116, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=697cd6a363e514bff0d4ec954fc4034b, ASSIGN; state=OFFLINE, location=037c58a19ddb,37267,1731751077844; forceNewPlan=false, retain=false 2024-11-16T09:59:53,153 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=117 updating hbase:meta row=697cd6a363e514bff0d4ec954fc4034b, regionState=OPENING, regionLocation=037c58a19ddb,37267,1731751077844 2024-11-16T09:59:53,155 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=117, ppid=116, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=697cd6a363e514bff0d4ec954fc4034b, ASSIGN because future has completed 2024-11-16T09:59:53,155 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=118, ppid=117, state=RUNNABLE, hasLock=false; OpenRegionProcedure 697cd6a363e514bff0d4ec954fc4034b, server=037c58a19ddb,37267,1731751077844}] 2024-11-16T09:59:53,309 INFO [RS_OPEN_REGION-regionserver/037c58a19ddb:0-0 {event_type=M_RS_OPEN_REGION, pid=118}] handler.AssignRegionHandler(132): Open TestAcidGuarantees,,1731751192181.697cd6a363e514bff0d4ec954fc4034b. 2024-11-16T09:59:53,310 DEBUG [RS_OPEN_REGION-regionserver/037c58a19ddb:0-0 {event_type=M_RS_OPEN_REGION, pid=118}] regionserver.HRegion(7752): Opening region: {ENCODED => 697cd6a363e514bff0d4ec954fc4034b, NAME => 'TestAcidGuarantees,,1731751192181.697cd6a363e514bff0d4ec954fc4034b.', STARTKEY => '', ENDKEY => ''} 2024-11-16T09:59:53,311 DEBUG [RS_OPEN_REGION-regionserver/037c58a19ddb:0-0 {event_type=M_RS_OPEN_REGION, pid=118}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestAcidGuarantees 697cd6a363e514bff0d4ec954fc4034b 2024-11-16T09:59:53,311 DEBUG [RS_OPEN_REGION-regionserver/037c58a19ddb:0-0 {event_type=M_RS_OPEN_REGION, pid=118}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1731751192181.697cd6a363e514bff0d4ec954fc4034b.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-16T09:59:53,311 DEBUG [RS_OPEN_REGION-regionserver/037c58a19ddb:0-0 {event_type=M_RS_OPEN_REGION, pid=118}] regionserver.HRegion(7794): checking encryption for 697cd6a363e514bff0d4ec954fc4034b 2024-11-16T09:59:53,311 DEBUG [RS_OPEN_REGION-regionserver/037c58a19ddb:0-0 {event_type=M_RS_OPEN_REGION, pid=118}] regionserver.HRegion(7797): checking classloading for 697cd6a363e514bff0d4ec954fc4034b 2024-11-16T09:59:53,312 INFO [StoreOpener-697cd6a363e514bff0d4ec954fc4034b-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family A of region 697cd6a363e514bff0d4ec954fc4034b 2024-11-16T09:59:53,313 INFO [StoreOpener-697cd6a363e514bff0d4ec954fc4034b-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 697cd6a363e514bff0d4ec954fc4034b columnFamilyName A 2024-11-16T09:59:53,313 DEBUG [StoreOpener-697cd6a363e514bff0d4ec954fc4034b-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T09:59:53,313 INFO [StoreOpener-697cd6a363e514bff0d4ec954fc4034b-1 {}] regionserver.HStore(327): Store=697cd6a363e514bff0d4ec954fc4034b/A, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-16T09:59:53,314 INFO [StoreOpener-697cd6a363e514bff0d4ec954fc4034b-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family B of region 697cd6a363e514bff0d4ec954fc4034b 2024-11-16T09:59:53,314 INFO [StoreOpener-697cd6a363e514bff0d4ec954fc4034b-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 697cd6a363e514bff0d4ec954fc4034b columnFamilyName B 2024-11-16T09:59:53,314 DEBUG [StoreOpener-697cd6a363e514bff0d4ec954fc4034b-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T09:59:53,315 INFO [StoreOpener-697cd6a363e514bff0d4ec954fc4034b-1 {}] regionserver.HStore(327): Store=697cd6a363e514bff0d4ec954fc4034b/B, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-16T09:59:53,315 INFO [StoreOpener-697cd6a363e514bff0d4ec954fc4034b-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family C of region 697cd6a363e514bff0d4ec954fc4034b 2024-11-16T09:59:53,316 INFO [StoreOpener-697cd6a363e514bff0d4ec954fc4034b-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 697cd6a363e514bff0d4ec954fc4034b columnFamilyName C 2024-11-16T09:59:53,316 DEBUG [StoreOpener-697cd6a363e514bff0d4ec954fc4034b-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T09:59:53,316 INFO [StoreOpener-697cd6a363e514bff0d4ec954fc4034b-1 {}] regionserver.HStore(327): Store=697cd6a363e514bff0d4ec954fc4034b/C, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-16T09:59:53,316 DEBUG [RS_OPEN_REGION-regionserver/037c58a19ddb:0-0 {event_type=M_RS_OPEN_REGION, pid=118}] regionserver.HRegion(1038): replaying wal for 697cd6a363e514bff0d4ec954fc4034b 2024-11-16T09:59:53,317 DEBUG [RS_OPEN_REGION-regionserver/037c58a19ddb:0-0 {event_type=M_RS_OPEN_REGION, pid=118}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/697cd6a363e514bff0d4ec954fc4034b 2024-11-16T09:59:53,317 DEBUG [RS_OPEN_REGION-regionserver/037c58a19ddb:0-0 {event_type=M_RS_OPEN_REGION, pid=118}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/697cd6a363e514bff0d4ec954fc4034b 2024-11-16T09:59:53,318 DEBUG [RS_OPEN_REGION-regionserver/037c58a19ddb:0-0 {event_type=M_RS_OPEN_REGION, pid=118}] regionserver.HRegion(1048): stopping wal replay for 697cd6a363e514bff0d4ec954fc4034b 2024-11-16T09:59:53,318 DEBUG [RS_OPEN_REGION-regionserver/037c58a19ddb:0-0 {event_type=M_RS_OPEN_REGION, pid=118}] regionserver.HRegion(1060): Cleaning up temporary data for 697cd6a363e514bff0d4ec954fc4034b 2024-11-16T09:59:53,318 DEBUG [RS_OPEN_REGION-regionserver/037c58a19ddb:0-0 {event_type=M_RS_OPEN_REGION, pid=118}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table TestAcidGuarantees descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-11-16T09:59:53,319 DEBUG [RS_OPEN_REGION-regionserver/037c58a19ddb:0-0 {event_type=M_RS_OPEN_REGION, pid=118}] regionserver.HRegion(1093): writing seq id for 697cd6a363e514bff0d4ec954fc4034b 2024-11-16T09:59:53,320 DEBUG [RS_OPEN_REGION-regionserver/037c58a19ddb:0-0 {event_type=M_RS_OPEN_REGION, pid=118}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/697cd6a363e514bff0d4ec954fc4034b/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-16T09:59:53,321 INFO [RS_OPEN_REGION-regionserver/037c58a19ddb:0-0 {event_type=M_RS_OPEN_REGION, pid=118}] regionserver.HRegion(1114): Opened 697cd6a363e514bff0d4ec954fc4034b; next sequenceid=2; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=69007888, jitterRate=0.02829766273498535}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-11-16T09:59:53,321 DEBUG [RS_OPEN_REGION-regionserver/037c58a19ddb:0-0 {event_type=M_RS_OPEN_REGION, pid=118}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 697cd6a363e514bff0d4ec954fc4034b 2024-11-16T09:59:53,321 DEBUG [RS_OPEN_REGION-regionserver/037c58a19ddb:0-0 {event_type=M_RS_OPEN_REGION, pid=118}] regionserver.HRegion(1006): Region open journal for 697cd6a363e514bff0d4ec954fc4034b: Running coprocessor pre-open hook at 1731751193311Writing region info on filesystem at 1731751193311Initializing all the Stores at 1731751193311Instantiating store for column family {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731751193312 (+1 ms)Instantiating store for column family {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731751193312Instantiating store for column family {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731751193312Cleaning up temporary data from old regions at 1731751193318 (+6 ms)Running coprocessor post-open hooks at 1731751193321 (+3 ms)Region opened successfully at 1731751193321 2024-11-16T09:59:53,322 INFO [RS_OPEN_REGION-regionserver/037c58a19ddb:0-0 {event_type=M_RS_OPEN_REGION, pid=118}] regionserver.HRegionServer(2236): Post open deploy tasks for TestAcidGuarantees,,1731751192181.697cd6a363e514bff0d4ec954fc4034b., pid=118, masterSystemTime=1731751193306 2024-11-16T09:59:53,323 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37877 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=116 2024-11-16T09:59:53,324 DEBUG [RS_OPEN_REGION-regionserver/037c58a19ddb:0-0 {event_type=M_RS_OPEN_REGION, pid=118}] regionserver.HRegionServer(2266): Finished post open deploy task for TestAcidGuarantees,,1731751192181.697cd6a363e514bff0d4ec954fc4034b. 2024-11-16T09:59:53,324 INFO [RS_OPEN_REGION-regionserver/037c58a19ddb:0-0 {event_type=M_RS_OPEN_REGION, pid=118}] handler.AssignRegionHandler(153): Opened TestAcidGuarantees,,1731751192181.697cd6a363e514bff0d4ec954fc4034b. 2024-11-16T09:59:53,324 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=117 updating hbase:meta row=697cd6a363e514bff0d4ec954fc4034b, regionState=OPEN, openSeqNum=2, regionLocation=037c58a19ddb,37267,1731751077844 2024-11-16T09:59:53,325 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=118, ppid=117, state=RUNNABLE, hasLock=false; OpenRegionProcedure 697cd6a363e514bff0d4ec954fc4034b, server=037c58a19ddb,37267,1731751077844 because future has completed 2024-11-16T09:59:53,327 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=118, resume processing ppid=117 2024-11-16T09:59:53,327 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=118, ppid=117, state=SUCCESS, hasLock=false; OpenRegionProcedure 697cd6a363e514bff0d4ec954fc4034b, server=037c58a19ddb,37267,1731751077844 in 171 msec 2024-11-16T09:59:53,329 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=117, resume processing ppid=116 2024-11-16T09:59:53,329 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=117, ppid=116, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=697cd6a363e514bff0d4ec954fc4034b, ASSIGN in 326 msec 2024-11-16T09:59:53,329 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=116, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-11-16T09:59:53,329 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731751193329"}]},"ts":"1731751193329"} 2024-11-16T09:59:53,330 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLED in hbase:meta 2024-11-16T09:59:53,331 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=116, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_POST_OPERATION 2024-11-16T09:59:53,332 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=116, state=SUCCESS, hasLock=false; CreateTableProcedure table=TestAcidGuarantees in 1.1500 sec 2024-11-16T09:59:54,333 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37877 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=116 2024-11-16T09:59:54,333 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: default:TestAcidGuarantees completed 2024-11-16T09:59:54,334 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@28ad78a3, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T09:59:54,334 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 037c58a19ddb,37877,-1 for getting cluster id 2024-11-16T09:59:54,334 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-16T09:59:54,335 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '1fe46ee8-cb44-44b1-8251-37103629f786' 2024-11-16T09:59:54,335 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-16T09:59:54,335 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "1fe46ee8-cb44-44b1-8251-37103629f786" 2024-11-16T09:59:54,335 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3d302deb, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T09:59:54,335 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [037c58a19ddb,37877,-1] 2024-11-16T09:59:54,336 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-16T09:59:54,336 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T09:59:54,336 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:49956, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-16T09:59:54,337 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3c2aa8da, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T09:59:54,337 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-16T09:59:54,338 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=037c58a19ddb,37267,1731751077844, seqNum=-1] 2024-11-16T09:59:54,338 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-16T09:59:54,339 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:35996, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-16T09:59:54,340 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-11-16T09:59:54,340 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncConnectionImpl(321): The fetched master address is 037c58a19ddb,37877,1731751077131 2024-11-16T09:59:54,340 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@7928e65c 2024-11-16T09:59:54,340 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-11-16T09:59:54,341 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:49968, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-11-16T09:59:54,342 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6ac69b7f, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T09:59:54,342 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 037c58a19ddb,37877,-1 for getting cluster id 2024-11-16T09:59:54,342 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-16T09:59:54,343 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '1fe46ee8-cb44-44b1-8251-37103629f786' 2024-11-16T09:59:54,343 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-16T09:59:54,343 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "1fe46ee8-cb44-44b1-8251-37103629f786" 2024-11-16T09:59:54,343 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@343f38b4, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T09:59:54,343 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [037c58a19ddb,37877,-1] 2024-11-16T09:59:54,343 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-16T09:59:54,344 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T09:59:54,344 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:49982, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-16T09:59:54,345 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5650bd2a, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T09:59:54,345 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@277a5c68, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T09:59:54,345 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 037c58a19ddb,37877,-1 for getting cluster id 2024-11-16T09:59:54,345 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-16T09:59:54,346 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '1fe46ee8-cb44-44b1-8251-37103629f786' 2024-11-16T09:59:54,346 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-16T09:59:54,346 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "1fe46ee8-cb44-44b1-8251-37103629f786" 2024-11-16T09:59:54,346 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4127b52f, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T09:59:54,346 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [037c58a19ddb,37877,-1] 2024-11-16T09:59:54,346 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-16T09:59:54,346 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T09:59:54,347 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:50002, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-16T09:59:54,347 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@18d418c0, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T09:59:54,348 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1df5cdc7, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T09:59:54,348 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 037c58a19ddb,37877,-1 for getting cluster id 2024-11-16T09:59:54,348 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-16T09:59:54,348 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '1fe46ee8-cb44-44b1-8251-37103629f786' 2024-11-16T09:59:54,348 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-16T09:59:54,348 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "1fe46ee8-cb44-44b1-8251-37103629f786" 2024-11-16T09:59:54,348 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@185f51bd, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T09:59:54,348 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [037c58a19ddb,37877,-1] 2024-11-16T09:59:54,349 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-16T09:59:54,349 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T09:59:54,349 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:50022, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-16T09:59:54,350 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7f5c77b5, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T09:59:54,350 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@22b9b902, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T09:59:54,350 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 037c58a19ddb,37877,-1 for getting cluster id 2024-11-16T09:59:54,350 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-16T09:59:54,351 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '1fe46ee8-cb44-44b1-8251-37103629f786' 2024-11-16T09:59:54,351 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-16T09:59:54,351 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "1fe46ee8-cb44-44b1-8251-37103629f786" 2024-11-16T09:59:54,351 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5440225f, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T09:59:54,351 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [037c58a19ddb,37877,-1] 2024-11-16T09:59:54,351 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-16T09:59:54,351 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T09:59:54,352 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:50036, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-16T09:59:54,352 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3d53a115, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T09:59:54,352 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3ff7674f, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T09:59:54,352 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 037c58a19ddb,37877,-1 for getting cluster id 2024-11-16T09:59:54,353 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-16T09:59:54,353 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '1fe46ee8-cb44-44b1-8251-37103629f786' 2024-11-16T09:59:54,353 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-16T09:59:54,353 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "1fe46ee8-cb44-44b1-8251-37103629f786" 2024-11-16T09:59:54,353 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3d68ad0a, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T09:59:54,353 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [037c58a19ddb,37877,-1] 2024-11-16T09:59:54,353 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-16T09:59:54,354 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T09:59:54,354 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:50058, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-16T09:59:54,354 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6fd8f81c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T09:59:54,355 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@107467c3, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T09:59:54,355 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 037c58a19ddb,37877,-1 for getting cluster id 2024-11-16T09:59:54,355 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-16T09:59:54,355 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '1fe46ee8-cb44-44b1-8251-37103629f786' 2024-11-16T09:59:54,356 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-16T09:59:54,356 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "1fe46ee8-cb44-44b1-8251-37103629f786" 2024-11-16T09:59:54,356 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@229770ea, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T09:59:54,356 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [037c58a19ddb,37877,-1] 2024-11-16T09:59:54,356 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-16T09:59:54,356 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T09:59:54,357 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:50078, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-16T09:59:54,357 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@545dee0e, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T09:59:54,358 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2acfd5e4, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T09:59:54,358 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 037c58a19ddb,37877,-1 for getting cluster id 2024-11-16T09:59:54,358 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-16T09:59:54,358 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '1fe46ee8-cb44-44b1-8251-37103629f786' 2024-11-16T09:59:54,358 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-16T09:59:54,358 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "1fe46ee8-cb44-44b1-8251-37103629f786" 2024-11-16T09:59:54,359 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3137f3d3, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T09:59:54,359 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [037c58a19ddb,37877,-1] 2024-11-16T09:59:54,359 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-16T09:59:54,359 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T09:59:54,359 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:50096, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-16T09:59:54,360 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1b5dbe8e, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T09:59:54,360 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@379767c9, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T09:59:54,360 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 037c58a19ddb,37877,-1 for getting cluster id 2024-11-16T09:59:54,360 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-16T09:59:54,361 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '1fe46ee8-cb44-44b1-8251-37103629f786' 2024-11-16T09:59:54,361 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-16T09:59:54,361 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "1fe46ee8-cb44-44b1-8251-37103629f786" 2024-11-16T09:59:54,361 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@17c085c3, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T09:59:54,361 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [037c58a19ddb,37877,-1] 2024-11-16T09:59:54,361 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-16T09:59:54,362 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T09:59:54,362 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:50110, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-16T09:59:54,363 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@38d2bbec, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T09:59:54,363 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@69f15306, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T09:59:54,363 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 037c58a19ddb,37877,-1 for getting cluster id 2024-11-16T09:59:54,363 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-16T09:59:54,364 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '1fe46ee8-cb44-44b1-8251-37103629f786' 2024-11-16T09:59:54,364 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-16T09:59:54,364 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "1fe46ee8-cb44-44b1-8251-37103629f786" 2024-11-16T09:59:54,364 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@20287f6b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T09:59:54,364 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [037c58a19ddb,37877,-1] 2024-11-16T09:59:54,364 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-16T09:59:54,364 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T09:59:54,365 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:50120, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-16T09:59:54,365 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@57761ec6, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T09:59:54,366 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4df5a2b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T09:59:54,366 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 037c58a19ddb,37877,-1 for getting cluster id 2024-11-16T09:59:54,366 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-16T09:59:54,366 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '1fe46ee8-cb44-44b1-8251-37103629f786' 2024-11-16T09:59:54,366 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-16T09:59:54,366 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "1fe46ee8-cb44-44b1-8251-37103629f786" 2024-11-16T09:59:54,367 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6d5c90bb, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T09:59:54,367 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [037c58a19ddb,37877,-1] 2024-11-16T09:59:54,367 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-16T09:59:54,367 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T09:59:54,367 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:50148, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-16T09:59:54,368 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2f736a9c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T09:59:54,368 DEBUG [Thread-2002 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-16T09:59:54,368 DEBUG [Thread-1991 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-16T09:59:54,368 DEBUG [Thread-1995 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-16T09:59:54,368 DEBUG [Thread-1997 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-16T09:59:54,369 DEBUG [Thread-2008 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-16T09:59:54,369 DEBUG [Thread-2006 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-16T09:59:54,369 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=037c58a19ddb,37267,1731751077844, seqNum=-1] 2024-11-16T09:59:54,369 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=037c58a19ddb,37267,1731751077844, seqNum=-1] 2024-11-16T09:59:54,369 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=037c58a19ddb,37267,1731751077844, seqNum=-1] 2024-11-16T09:59:54,369 DEBUG [Thread-2004 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-16T09:59:54,369 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-16T09:59:54,369 DEBUG [Thread-1993 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-16T09:59:54,369 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-16T09:59:54,369 DEBUG [Thread-1989 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-16T09:59:54,369 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-16T09:59:54,370 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:36012, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-16T09:59:54,370 DEBUG [Thread-2000 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-16T09:59:54,370 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=037c58a19ddb,37267,1731751077844, seqNum=-1] 2024-11-16T09:59:54,370 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=037c58a19ddb,37267,1731751077844, seqNum=-1] 2024-11-16T09:59:54,370 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:36018, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-16T09:59:54,371 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:36022, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-16T09:59:54,371 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-16T09:59:54,371 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-16T09:59:54,371 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=037c58a19ddb,37267,1731751077844, seqNum=-1] 2024-11-16T09:59:54,371 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-16T09:59:54,371 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=037c58a19ddb,37267,1731751077844, seqNum=-1] 2024-11-16T09:59:54,371 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-16T09:59:54,371 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:36034, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-16T09:59:54,372 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:36046, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-16T09:59:54,372 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=037c58a19ddb,37267,1731751077844, seqNum=-1] 2024-11-16T09:59:54,372 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-16T09:59:54,372 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:36058, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-16T09:59:54,372 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:36068, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-16T09:59:54,373 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=037c58a19ddb,37267,1731751077844, seqNum=-1] 2024-11-16T09:59:54,373 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-16T09:59:54,373 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:36072, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-16T09:59:54,374 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1731751192181.697cd6a363e514bff0d4ec954fc4034b., hostname=037c58a19ddb,37267,1731751077844, seqNum=2] 2024-11-16T09:59:54,374 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1731751192181.697cd6a363e514bff0d4ec954fc4034b., hostname=037c58a19ddb,37267,1731751077844, seqNum=2] 2024-11-16T09:59:54,374 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37877 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-16T09:59:54,374 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:36074, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-16T09:59:54,374 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1731751192181.697cd6a363e514bff0d4ec954fc4034b., hostname=037c58a19ddb,37267,1731751077844, seqNum=2] 2024-11-16T09:59:54,374 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=037c58a19ddb,37267,1731751077844, seqNum=-1] 2024-11-16T09:59:54,375 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-16T09:59:54,375 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1731751192181.697cd6a363e514bff0d4ec954fc4034b., hostname=037c58a19ddb,37267,1731751077844, seqNum=2] 2024-11-16T09:59:54,375 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1731751192181.697cd6a363e514bff0d4ec954fc4034b., hostname=037c58a19ddb,37267,1731751077844, seqNum=2] 2024-11-16T09:59:54,376 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1731751192181.697cd6a363e514bff0d4ec954fc4034b., hostname=037c58a19ddb,37267,1731751077844, seqNum=2] 2024-11-16T09:59:54,376 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37877 {}] procedure2.ProcedureExecutor(1139): Stored pid=119, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=119, table=TestAcidGuarantees 2024-11-16T09:59:54,376 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1731751192181.697cd6a363e514bff0d4ec954fc4034b., hostname=037c58a19ddb,37267,1731751077844, seqNum=2] 2024-11-16T09:59:54,376 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1731751192181.697cd6a363e514bff0d4ec954fc4034b., hostname=037c58a19ddb,37267,1731751077844, seqNum=2] 2024-11-16T09:59:54,376 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1731751192181.697cd6a363e514bff0d4ec954fc4034b., hostname=037c58a19ddb,37267,1731751077844, seqNum=2] 2024-11-16T09:59:54,377 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37877 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=119 2024-11-16T09:59:54,377 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:36086, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-16T09:59:54,378 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=119, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=119, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-16T09:59:54,378 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=119, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=119, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-16T09:59:54,379 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=120, ppid=119, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-16T09:59:54,379 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1731751192181.697cd6a363e514bff0d4ec954fc4034b., hostname=037c58a19ddb,37267,1731751077844, seqNum=2] 2024-11-16T09:59:54,390 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] regionserver.HRegion(8855): Flush requested on 697cd6a363e514bff0d4ec954fc4034b 2024-11-16T09:59:54,390 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-16T09:59:54,390 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 697cd6a363e514bff0d4ec954fc4034b 3/3 column families, dataSize=53.67 KB heapSize=141.38 KB 2024-11-16T09:59:54,419 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/697cd6a363e514bff0d4ec954fc4034b/.tmp/A/656d8bd3c1f740f590bef993761e4820 is 50, key is test_row_0/A:col10/1731751194376/Put/seqid=0 2024-11-16T09:59:54,429 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37591 is added to blk_1073742268_1444 (size=12001) 2024-11-16T09:59:54,483 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37877 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=119 2024-11-16T09:59:54,487 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=697cd6a363e514bff0d4ec954fc4034b, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T09:59:54,487 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36012 deadline: 1731751254469, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=697cd6a363e514bff0d4ec954fc4034b, server=037c58a19ddb,37267,1731751077844 2024-11-16T09:59:54,488 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=697cd6a363e514bff0d4ec954fc4034b, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T09:59:54,488 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731751192181.697cd6a363e514bff0d4ec954fc4034b., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 , the old value is region=TestAcidGuarantees,,1731751192181.697cd6a363e514bff0d4ec954fc4034b., hostname=037c58a19ddb,37267,1731751077844, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=697cd6a363e514bff0d4ec954fc4034b, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:59:54,488 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731751192181.697cd6a363e514bff0d4ec954fc4034b., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=697cd6a363e514bff0d4ec954fc4034b, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:59:54,488 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36058 deadline: 1731751254470, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=697cd6a363e514bff0d4ec954fc4034b, server=037c58a19ddb,37267,1731751077844 2024-11-16T09:59:54,488 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731751192181.697cd6a363e514bff0d4ec954fc4034b., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 because the exception is null or not the one we care about 2024-11-16T09:59:54,488 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731751192181.697cd6a363e514bff0d4ec954fc4034b., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 , the old value is region=TestAcidGuarantees,,1731751192181.697cd6a363e514bff0d4ec954fc4034b., hostname=037c58a19ddb,37267,1731751077844, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=697cd6a363e514bff0d4ec954fc4034b, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:59:54,488 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=697cd6a363e514bff0d4ec954fc4034b, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T09:59:54,488 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731751192181.697cd6a363e514bff0d4ec954fc4034b., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=697cd6a363e514bff0d4ec954fc4034b, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:59:54,488 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731751192181.697cd6a363e514bff0d4ec954fc4034b., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 because the exception is null or not the one we care about 2024-11-16T09:59:54,488 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] ipc.CallRunner(138): callId: 6 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36086 deadline: 1731751254471, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=697cd6a363e514bff0d4ec954fc4034b, server=037c58a19ddb,37267,1731751077844 2024-11-16T09:59:54,489 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731751192181.697cd6a363e514bff0d4ec954fc4034b., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 , the old value is region=TestAcidGuarantees,,1731751192181.697cd6a363e514bff0d4ec954fc4034b., hostname=037c58a19ddb,37267,1731751077844, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=697cd6a363e514bff0d4ec954fc4034b, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:59:54,489 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731751192181.697cd6a363e514bff0d4ec954fc4034b., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=697cd6a363e514bff0d4ec954fc4034b, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:59:54,489 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731751192181.697cd6a363e514bff0d4ec954fc4034b., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 because the exception is null or not the one we care about 2024-11-16T09:59:54,489 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=697cd6a363e514bff0d4ec954fc4034b, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T09:59:54,489 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36046 deadline: 1731751254474, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=697cd6a363e514bff0d4ec954fc4034b, server=037c58a19ddb,37267,1731751077844 2024-11-16T09:59:54,489 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731751192181.697cd6a363e514bff0d4ec954fc4034b., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 , the old value is region=TestAcidGuarantees,,1731751192181.697cd6a363e514bff0d4ec954fc4034b., hostname=037c58a19ddb,37267,1731751077844, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=697cd6a363e514bff0d4ec954fc4034b, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:59:54,489 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731751192181.697cd6a363e514bff0d4ec954fc4034b., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=697cd6a363e514bff0d4ec954fc4034b, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:59:54,489 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731751192181.697cd6a363e514bff0d4ec954fc4034b., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 because the exception is null or not the one we care about 2024-11-16T09:59:54,500 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=697cd6a363e514bff0d4ec954fc4034b, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T09:59:54,501 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36022 deadline: 1731751254487, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=697cd6a363e514bff0d4ec954fc4034b, server=037c58a19ddb,37267,1731751077844 2024-11-16T09:59:54,501 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731751192181.697cd6a363e514bff0d4ec954fc4034b., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 , the old value is region=TestAcidGuarantees,,1731751192181.697cd6a363e514bff0d4ec954fc4034b., hostname=037c58a19ddb,37267,1731751077844, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=697cd6a363e514bff0d4ec954fc4034b, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:59:54,501 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731751192181.697cd6a363e514bff0d4ec954fc4034b., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=697cd6a363e514bff0d4ec954fc4034b, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:59:54,501 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731751192181.697cd6a363e514bff0d4ec954fc4034b., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 because the exception is null or not the one we care about 2024-11-16T09:59:54,531 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=37267 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=120 2024-11-16T09:59:54,531 DEBUG [RS_FLUSH_OPERATIONS-regionserver/037c58a19ddb:0-2 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731751192181.697cd6a363e514bff0d4ec954fc4034b. 2024-11-16T09:59:54,531 DEBUG [RS_FLUSH_OPERATIONS-regionserver/037c58a19ddb:0-2 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731751192181.697cd6a363e514bff0d4ec954fc4034b. as already flushing 2024-11-16T09:59:54,531 DEBUG [RS_FLUSH_OPERATIONS-regionserver/037c58a19ddb:0-2 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731751192181.697cd6a363e514bff0d4ec954fc4034b. 2024-11-16T09:59:54,532 ERROR [RS_FLUSH_OPERATIONS-regionserver/037c58a19ddb:0-2 {event_type=RS_FLUSH_REGIONS, pid=120}] handler.RSProcedureHandler(58): pid=120 java.io.IOException: Unable to complete flush {ENCODED => 697cd6a363e514bff0d4ec954fc4034b, NAME => 'TestAcidGuarantees,,1731751192181.697cd6a363e514bff0d4ec954fc4034b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T09:59:54,532 DEBUG [RS_FLUSH_OPERATIONS-regionserver/037c58a19ddb:0-2 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=120 java.io.IOException: Unable to complete flush {ENCODED => 697cd6a363e514bff0d4ec954fc4034b, NAME => 'TestAcidGuarantees,,1731751192181.697cd6a363e514bff0d4ec954fc4034b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T09:59:54,532 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37877 {}] master.HMaster(4177): Remote procedure failed, pid=120 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 697cd6a363e514bff0d4ec954fc4034b, NAME => 'TestAcidGuarantees,,1731751192181.697cd6a363e514bff0d4ec954fc4034b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 697cd6a363e514bff0d4ec954fc4034b, NAME => 'TestAcidGuarantees,,1731751192181.697cd6a363e514bff0d4ec954fc4034b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T09:59:54,600 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=697cd6a363e514bff0d4ec954fc4034b, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T09:59:54,600 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36012 deadline: 1731751254592, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=697cd6a363e514bff0d4ec954fc4034b, server=037c58a19ddb,37267,1731751077844 2024-11-16T09:59:54,601 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731751192181.697cd6a363e514bff0d4ec954fc4034b., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 , the old value is region=TestAcidGuarantees,,1731751192181.697cd6a363e514bff0d4ec954fc4034b., hostname=037c58a19ddb,37267,1731751077844, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=697cd6a363e514bff0d4ec954fc4034b, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:59:54,601 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731751192181.697cd6a363e514bff0d4ec954fc4034b., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=697cd6a363e514bff0d4ec954fc4034b, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:59:54,601 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=697cd6a363e514bff0d4ec954fc4034b, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T09:59:54,601 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731751192181.697cd6a363e514bff0d4ec954fc4034b., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 because the exception is null or not the one we care about 2024-11-16T09:59:54,601 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36058 deadline: 1731751254593, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=697cd6a363e514bff0d4ec954fc4034b, server=037c58a19ddb,37267,1731751077844 2024-11-16T09:59:54,601 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731751192181.697cd6a363e514bff0d4ec954fc4034b., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 , the old value is region=TestAcidGuarantees,,1731751192181.697cd6a363e514bff0d4ec954fc4034b., hostname=037c58a19ddb,37267,1731751077844, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=697cd6a363e514bff0d4ec954fc4034b, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:59:54,601 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731751192181.697cd6a363e514bff0d4ec954fc4034b., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=697cd6a363e514bff0d4ec954fc4034b, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:59:54,601 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731751192181.697cd6a363e514bff0d4ec954fc4034b., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 because the exception is null or not the one we care about 2024-11-16T09:59:54,601 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=697cd6a363e514bff0d4ec954fc4034b, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T09:59:54,601 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36086 deadline: 1731751254593, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=697cd6a363e514bff0d4ec954fc4034b, server=037c58a19ddb,37267,1731751077844 2024-11-16T09:59:54,601 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731751192181.697cd6a363e514bff0d4ec954fc4034b., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 , the old value is region=TestAcidGuarantees,,1731751192181.697cd6a363e514bff0d4ec954fc4034b., hostname=037c58a19ddb,37267,1731751077844, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=697cd6a363e514bff0d4ec954fc4034b, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:59:54,601 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=697cd6a363e514bff0d4ec954fc4034b, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T09:59:54,602 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731751192181.697cd6a363e514bff0d4ec954fc4034b., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=697cd6a363e514bff0d4ec954fc4034b, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:59:54,602 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731751192181.697cd6a363e514bff0d4ec954fc4034b., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 because the exception is null or not the one we care about 2024-11-16T09:59:54,602 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36046 deadline: 1731751254593, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=697cd6a363e514bff0d4ec954fc4034b, server=037c58a19ddb,37267,1731751077844 2024-11-16T09:59:54,602 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731751192181.697cd6a363e514bff0d4ec954fc4034b., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 , the old value is region=TestAcidGuarantees,,1731751192181.697cd6a363e514bff0d4ec954fc4034b., hostname=037c58a19ddb,37267,1731751077844, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=697cd6a363e514bff0d4ec954fc4034b, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:59:54,602 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731751192181.697cd6a363e514bff0d4ec954fc4034b., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=697cd6a363e514bff0d4ec954fc4034b, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:59:54,602 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731751192181.697cd6a363e514bff0d4ec954fc4034b., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 because the exception is null or not the one we care about 2024-11-16T09:59:54,611 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=697cd6a363e514bff0d4ec954fc4034b, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T09:59:54,612 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36022 deadline: 1731751254603, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=697cd6a363e514bff0d4ec954fc4034b, server=037c58a19ddb,37267,1731751077844 2024-11-16T09:59:54,612 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731751192181.697cd6a363e514bff0d4ec954fc4034b., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 , the old value is region=TestAcidGuarantees,,1731751192181.697cd6a363e514bff0d4ec954fc4034b., hostname=037c58a19ddb,37267,1731751077844, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=697cd6a363e514bff0d4ec954fc4034b, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:59:54,612 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731751192181.697cd6a363e514bff0d4ec954fc4034b., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=697cd6a363e514bff0d4ec954fc4034b, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:59:54,612 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731751192181.697cd6a363e514bff0d4ec954fc4034b., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 because the exception is null or not the one we care about 2024-11-16T09:59:54,684 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=37267 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=120 2024-11-16T09:59:54,684 DEBUG [RS_FLUSH_OPERATIONS-regionserver/037c58a19ddb:0-0 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731751192181.697cd6a363e514bff0d4ec954fc4034b. 2024-11-16T09:59:54,684 DEBUG [RS_FLUSH_OPERATIONS-regionserver/037c58a19ddb:0-0 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731751192181.697cd6a363e514bff0d4ec954fc4034b. as already flushing 2024-11-16T09:59:54,684 DEBUG [RS_FLUSH_OPERATIONS-regionserver/037c58a19ddb:0-0 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731751192181.697cd6a363e514bff0d4ec954fc4034b. 2024-11-16T09:59:54,685 ERROR [RS_FLUSH_OPERATIONS-regionserver/037c58a19ddb:0-0 {event_type=RS_FLUSH_REGIONS, pid=120}] handler.RSProcedureHandler(58): pid=120 java.io.IOException: Unable to complete flush {ENCODED => 697cd6a363e514bff0d4ec954fc4034b, NAME => 'TestAcidGuarantees,,1731751192181.697cd6a363e514bff0d4ec954fc4034b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T09:59:54,685 DEBUG [RS_FLUSH_OPERATIONS-regionserver/037c58a19ddb:0-0 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=120 java.io.IOException: Unable to complete flush {ENCODED => 697cd6a363e514bff0d4ec954fc4034b, NAME => 'TestAcidGuarantees,,1731751192181.697cd6a363e514bff0d4ec954fc4034b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T09:59:54,685 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37877 {}] master.HMaster(4177): Remote procedure failed, pid=120 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 697cd6a363e514bff0d4ec954fc4034b, NAME => 'TestAcidGuarantees,,1731751192181.697cd6a363e514bff0d4ec954fc4034b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 697cd6a363e514bff0d4ec954fc4034b, NAME => 'TestAcidGuarantees,,1731751192181.697cd6a363e514bff0d4ec954fc4034b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T09:59:54,693 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37877 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=119 2024-11-16T09:59:54,820 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=697cd6a363e514bff0d4ec954fc4034b, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T09:59:54,820 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36012 deadline: 1731751254813, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=697cd6a363e514bff0d4ec954fc4034b, server=037c58a19ddb,37267,1731751077844 2024-11-16T09:59:54,820 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=697cd6a363e514bff0d4ec954fc4034b, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T09:59:54,820 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731751192181.697cd6a363e514bff0d4ec954fc4034b., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 , the old value is region=TestAcidGuarantees,,1731751192181.697cd6a363e514bff0d4ec954fc4034b., hostname=037c58a19ddb,37267,1731751077844, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=697cd6a363e514bff0d4ec954fc4034b, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:59:54,820 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731751192181.697cd6a363e514bff0d4ec954fc4034b., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=697cd6a363e514bff0d4ec954fc4034b, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:59:54,820 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36058 deadline: 1731751254813, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=697cd6a363e514bff0d4ec954fc4034b, server=037c58a19ddb,37267,1731751077844 2024-11-16T09:59:54,820 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731751192181.697cd6a363e514bff0d4ec954fc4034b., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 because the exception is null or not the one we care about 2024-11-16T09:59:54,821 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731751192181.697cd6a363e514bff0d4ec954fc4034b., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 , the old value is region=TestAcidGuarantees,,1731751192181.697cd6a363e514bff0d4ec954fc4034b., hostname=037c58a19ddb,37267,1731751077844, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=697cd6a363e514bff0d4ec954fc4034b, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:59:54,821 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731751192181.697cd6a363e514bff0d4ec954fc4034b., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=697cd6a363e514bff0d4ec954fc4034b, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:59:54,821 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=697cd6a363e514bff0d4ec954fc4034b, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T09:59:54,821 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731751192181.697cd6a363e514bff0d4ec954fc4034b., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 because the exception is null or not the one we care about 2024-11-16T09:59:54,821 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36086 deadline: 1731751254813, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=697cd6a363e514bff0d4ec954fc4034b, server=037c58a19ddb,37267,1731751077844 2024-11-16T09:59:54,821 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731751192181.697cd6a363e514bff0d4ec954fc4034b., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 , the old value is region=TestAcidGuarantees,,1731751192181.697cd6a363e514bff0d4ec954fc4034b., hostname=037c58a19ddb,37267,1731751077844, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=697cd6a363e514bff0d4ec954fc4034b, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:59:54,821 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731751192181.697cd6a363e514bff0d4ec954fc4034b., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=697cd6a363e514bff0d4ec954fc4034b, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:59:54,821 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731751192181.697cd6a363e514bff0d4ec954fc4034b., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 because the exception is null or not the one we care about 2024-11-16T09:59:54,821 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=697cd6a363e514bff0d4ec954fc4034b, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T09:59:54,821 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36046 deadline: 1731751254813, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=697cd6a363e514bff0d4ec954fc4034b, server=037c58a19ddb,37267,1731751077844 2024-11-16T09:59:54,821 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731751192181.697cd6a363e514bff0d4ec954fc4034b., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 , the old value is region=TestAcidGuarantees,,1731751192181.697cd6a363e514bff0d4ec954fc4034b., hostname=037c58a19ddb,37267,1731751077844, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=697cd6a363e514bff0d4ec954fc4034b, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:59:54,821 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731751192181.697cd6a363e514bff0d4ec954fc4034b., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=697cd6a363e514bff0d4ec954fc4034b, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:59:54,821 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731751192181.697cd6a363e514bff0d4ec954fc4034b., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 because the exception is null or not the one we care about 2024-11-16T09:59:54,830 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=697cd6a363e514bff0d4ec954fc4034b, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T09:59:54,830 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36022 deadline: 1731751254822, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=697cd6a363e514bff0d4ec954fc4034b, server=037c58a19ddb,37267,1731751077844 2024-11-16T09:59:54,830 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=12 (bloomFilter=true), to=hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/697cd6a363e514bff0d4ec954fc4034b/.tmp/A/656d8bd3c1f740f590bef993761e4820 2024-11-16T09:59:54,830 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731751192181.697cd6a363e514bff0d4ec954fc4034b., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 , the old value is region=TestAcidGuarantees,,1731751192181.697cd6a363e514bff0d4ec954fc4034b., hostname=037c58a19ddb,37267,1731751077844, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=697cd6a363e514bff0d4ec954fc4034b, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:59:54,830 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731751192181.697cd6a363e514bff0d4ec954fc4034b., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=697cd6a363e514bff0d4ec954fc4034b, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:59:54,830 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731751192181.697cd6a363e514bff0d4ec954fc4034b., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 because the exception is null or not the one we care about 2024-11-16T09:59:54,837 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=37267 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=120 2024-11-16T09:59:54,837 DEBUG [RS_FLUSH_OPERATIONS-regionserver/037c58a19ddb:0-1 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731751192181.697cd6a363e514bff0d4ec954fc4034b. 2024-11-16T09:59:54,837 DEBUG [RS_FLUSH_OPERATIONS-regionserver/037c58a19ddb:0-1 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731751192181.697cd6a363e514bff0d4ec954fc4034b. as already flushing 2024-11-16T09:59:54,838 DEBUG [RS_FLUSH_OPERATIONS-regionserver/037c58a19ddb:0-1 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731751192181.697cd6a363e514bff0d4ec954fc4034b. 2024-11-16T09:59:54,838 ERROR [RS_FLUSH_OPERATIONS-regionserver/037c58a19ddb:0-1 {event_type=RS_FLUSH_REGIONS, pid=120}] handler.RSProcedureHandler(58): pid=120 java.io.IOException: Unable to complete flush {ENCODED => 697cd6a363e514bff0d4ec954fc4034b, NAME => 'TestAcidGuarantees,,1731751192181.697cd6a363e514bff0d4ec954fc4034b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T09:59:54,838 DEBUG [RS_FLUSH_OPERATIONS-regionserver/037c58a19ddb:0-1 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=120 java.io.IOException: Unable to complete flush {ENCODED => 697cd6a363e514bff0d4ec954fc4034b, NAME => 'TestAcidGuarantees,,1731751192181.697cd6a363e514bff0d4ec954fc4034b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T09:59:54,838 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37877 {}] master.HMaster(4177): Remote procedure failed, pid=120 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 697cd6a363e514bff0d4ec954fc4034b, NAME => 'TestAcidGuarantees,,1731751192181.697cd6a363e514bff0d4ec954fc4034b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 697cd6a363e514bff0d4ec954fc4034b, NAME => 'TestAcidGuarantees,,1731751192181.697cd6a363e514bff0d4ec954fc4034b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T09:59:54,855 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/697cd6a363e514bff0d4ec954fc4034b/.tmp/B/b879c013fa994e78b1fe7c6327ecd121 is 50, key is test_row_0/B:col10/1731751194376/Put/seqid=0 2024-11-16T09:59:54,859 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37591 is added to blk_1073742269_1445 (size=12001) 2024-11-16T09:59:54,990 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=37267 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=120 2024-11-16T09:59:54,990 DEBUG [RS_FLUSH_OPERATIONS-regionserver/037c58a19ddb:0-2 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731751192181.697cd6a363e514bff0d4ec954fc4034b. 2024-11-16T09:59:54,990 DEBUG [RS_FLUSH_OPERATIONS-regionserver/037c58a19ddb:0-2 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731751192181.697cd6a363e514bff0d4ec954fc4034b. as already flushing 2024-11-16T09:59:54,990 DEBUG [RS_FLUSH_OPERATIONS-regionserver/037c58a19ddb:0-2 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731751192181.697cd6a363e514bff0d4ec954fc4034b. 2024-11-16T09:59:54,991 ERROR [RS_FLUSH_OPERATIONS-regionserver/037c58a19ddb:0-2 {event_type=RS_FLUSH_REGIONS, pid=120}] handler.RSProcedureHandler(58): pid=120 java.io.IOException: Unable to complete flush {ENCODED => 697cd6a363e514bff0d4ec954fc4034b, NAME => 'TestAcidGuarantees,,1731751192181.697cd6a363e514bff0d4ec954fc4034b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T09:59:54,991 DEBUG [RS_FLUSH_OPERATIONS-regionserver/037c58a19ddb:0-2 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=120 java.io.IOException: Unable to complete flush {ENCODED => 697cd6a363e514bff0d4ec954fc4034b, NAME => 'TestAcidGuarantees,,1731751192181.697cd6a363e514bff0d4ec954fc4034b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T09:59:54,991 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37877 {}] master.HMaster(4177): Remote procedure failed, pid=120 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 697cd6a363e514bff0d4ec954fc4034b, NAME => 'TestAcidGuarantees,,1731751192181.697cd6a363e514bff0d4ec954fc4034b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 697cd6a363e514bff0d4ec954fc4034b, NAME => 'TestAcidGuarantees,,1731751192181.697cd6a363e514bff0d4ec954fc4034b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T09:59:55,003 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37877 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=119 2024-11-16T09:59:55,129 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=697cd6a363e514bff0d4ec954fc4034b, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T09:59:55,130 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36086 deadline: 1731751255123, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=697cd6a363e514bff0d4ec954fc4034b, server=037c58a19ddb,37267,1731751077844 2024-11-16T09:59:55,130 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731751192181.697cd6a363e514bff0d4ec954fc4034b., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 , the old value is region=TestAcidGuarantees,,1731751192181.697cd6a363e514bff0d4ec954fc4034b., hostname=037c58a19ddb,37267,1731751077844, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=697cd6a363e514bff0d4ec954fc4034b, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:59:55,130 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731751192181.697cd6a363e514bff0d4ec954fc4034b., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=697cd6a363e514bff0d4ec954fc4034b, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:59:55,130 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731751192181.697cd6a363e514bff0d4ec954fc4034b., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 because the exception is null or not the one we care about 2024-11-16T09:59:55,138 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=697cd6a363e514bff0d4ec954fc4034b, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T09:59:55,138 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36012 deadline: 1731751255133, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=697cd6a363e514bff0d4ec954fc4034b, server=037c58a19ddb,37267,1731751077844 2024-11-16T09:59:55,139 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=697cd6a363e514bff0d4ec954fc4034b, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T09:59:55,139 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731751192181.697cd6a363e514bff0d4ec954fc4034b., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 , the old value is region=TestAcidGuarantees,,1731751192181.697cd6a363e514bff0d4ec954fc4034b., hostname=037c58a19ddb,37267,1731751077844, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=697cd6a363e514bff0d4ec954fc4034b, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:59:55,139 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731751192181.697cd6a363e514bff0d4ec954fc4034b., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=697cd6a363e514bff0d4ec954fc4034b, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:59:55,139 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731751192181.697cd6a363e514bff0d4ec954fc4034b., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 because the exception is null or not the one we care about 2024-11-16T09:59:55,139 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36058 deadline: 1731751255133, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=697cd6a363e514bff0d4ec954fc4034b, server=037c58a19ddb,37267,1731751077844 2024-11-16T09:59:55,139 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=697cd6a363e514bff0d4ec954fc4034b, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T09:59:55,139 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731751192181.697cd6a363e514bff0d4ec954fc4034b., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 , the old value is region=TestAcidGuarantees,,1731751192181.697cd6a363e514bff0d4ec954fc4034b., hostname=037c58a19ddb,37267,1731751077844, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=697cd6a363e514bff0d4ec954fc4034b, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:59:55,139 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731751192181.697cd6a363e514bff0d4ec954fc4034b., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=697cd6a363e514bff0d4ec954fc4034b, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:59:55,139 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731751192181.697cd6a363e514bff0d4ec954fc4034b., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 because the exception is null or not the one we care about 2024-11-16T09:59:55,139 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36046 deadline: 1731751255133, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=697cd6a363e514bff0d4ec954fc4034b, server=037c58a19ddb,37267,1731751077844 2024-11-16T09:59:55,139 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731751192181.697cd6a363e514bff0d4ec954fc4034b., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 , the old value is region=TestAcidGuarantees,,1731751192181.697cd6a363e514bff0d4ec954fc4034b., hostname=037c58a19ddb,37267,1731751077844, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=697cd6a363e514bff0d4ec954fc4034b, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:59:55,139 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=697cd6a363e514bff0d4ec954fc4034b, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T09:59:55,139 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731751192181.697cd6a363e514bff0d4ec954fc4034b., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=697cd6a363e514bff0d4ec954fc4034b, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:59:55,139 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731751192181.697cd6a363e514bff0d4ec954fc4034b., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 because the exception is null or not the one we care about 2024-11-16T09:59:55,139 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36022 deadline: 1731751255134, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=697cd6a363e514bff0d4ec954fc4034b, server=037c58a19ddb,37267,1731751077844 2024-11-16T09:59:55,139 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731751192181.697cd6a363e514bff0d4ec954fc4034b., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 , the old value is region=TestAcidGuarantees,,1731751192181.697cd6a363e514bff0d4ec954fc4034b., hostname=037c58a19ddb,37267,1731751077844, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=697cd6a363e514bff0d4ec954fc4034b, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:59:55,139 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731751192181.697cd6a363e514bff0d4ec954fc4034b., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=697cd6a363e514bff0d4ec954fc4034b, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:59:55,139 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731751192181.697cd6a363e514bff0d4ec954fc4034b., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 because the exception is null or not the one we care about 2024-11-16T09:59:55,143 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=37267 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=120 2024-11-16T09:59:55,143 DEBUG [RS_FLUSH_OPERATIONS-regionserver/037c58a19ddb:0-0 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731751192181.697cd6a363e514bff0d4ec954fc4034b. 2024-11-16T09:59:55,143 DEBUG [RS_FLUSH_OPERATIONS-regionserver/037c58a19ddb:0-0 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731751192181.697cd6a363e514bff0d4ec954fc4034b. as already flushing 2024-11-16T09:59:55,143 DEBUG [RS_FLUSH_OPERATIONS-regionserver/037c58a19ddb:0-0 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731751192181.697cd6a363e514bff0d4ec954fc4034b. 2024-11-16T09:59:55,143 ERROR [RS_FLUSH_OPERATIONS-regionserver/037c58a19ddb:0-0 {event_type=RS_FLUSH_REGIONS, pid=120}] handler.RSProcedureHandler(58): pid=120 java.io.IOException: Unable to complete flush {ENCODED => 697cd6a363e514bff0d4ec954fc4034b, NAME => 'TestAcidGuarantees,,1731751192181.697cd6a363e514bff0d4ec954fc4034b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T09:59:55,144 DEBUG [RS_FLUSH_OPERATIONS-regionserver/037c58a19ddb:0-0 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=120 java.io.IOException: Unable to complete flush {ENCODED => 697cd6a363e514bff0d4ec954fc4034b, NAME => 'TestAcidGuarantees,,1731751192181.697cd6a363e514bff0d4ec954fc4034b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T09:59:55,144 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37877 {}] master.HMaster(4177): Remote procedure failed, pid=120 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 697cd6a363e514bff0d4ec954fc4034b, NAME => 'TestAcidGuarantees,,1731751192181.697cd6a363e514bff0d4ec954fc4034b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 697cd6a363e514bff0d4ec954fc4034b, NAME => 'TestAcidGuarantees,,1731751192181.697cd6a363e514bff0d4ec954fc4034b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T09:59:55,260 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=12 (bloomFilter=true), to=hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/697cd6a363e514bff0d4ec954fc4034b/.tmp/B/b879c013fa994e78b1fe7c6327ecd121 2024-11-16T09:59:55,284 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/697cd6a363e514bff0d4ec954fc4034b/.tmp/C/a5314ebc7f464faba37fa36b02402253 is 50, key is test_row_0/C:col10/1731751194376/Put/seqid=0 2024-11-16T09:59:55,287 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37591 is added to blk_1073742270_1446 (size=12001) 2024-11-16T09:59:55,296 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=37267 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=120 2024-11-16T09:59:55,296 DEBUG [RS_FLUSH_OPERATIONS-regionserver/037c58a19ddb:0-1 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731751192181.697cd6a363e514bff0d4ec954fc4034b. 2024-11-16T09:59:55,296 DEBUG [RS_FLUSH_OPERATIONS-regionserver/037c58a19ddb:0-1 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731751192181.697cd6a363e514bff0d4ec954fc4034b. as already flushing 2024-11-16T09:59:55,296 DEBUG [RS_FLUSH_OPERATIONS-regionserver/037c58a19ddb:0-1 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731751192181.697cd6a363e514bff0d4ec954fc4034b. 2024-11-16T09:59:55,297 ERROR [RS_FLUSH_OPERATIONS-regionserver/037c58a19ddb:0-1 {event_type=RS_FLUSH_REGIONS, pid=120}] handler.RSProcedureHandler(58): pid=120 java.io.IOException: Unable to complete flush {ENCODED => 697cd6a363e514bff0d4ec954fc4034b, NAME => 'TestAcidGuarantees,,1731751192181.697cd6a363e514bff0d4ec954fc4034b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T09:59:55,297 DEBUG [RS_FLUSH_OPERATIONS-regionserver/037c58a19ddb:0-1 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=120 java.io.IOException: Unable to complete flush {ENCODED => 697cd6a363e514bff0d4ec954fc4034b, NAME => 'TestAcidGuarantees,,1731751192181.697cd6a363e514bff0d4ec954fc4034b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T09:59:55,297 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37877 {}] master.HMaster(4177): Remote procedure failed, pid=120 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 697cd6a363e514bff0d4ec954fc4034b, NAME => 'TestAcidGuarantees,,1731751192181.697cd6a363e514bff0d4ec954fc4034b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 697cd6a363e514bff0d4ec954fc4034b, NAME => 'TestAcidGuarantees,,1731751192181.697cd6a363e514bff0d4ec954fc4034b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T09:59:55,449 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=37267 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=120 2024-11-16T09:59:55,450 DEBUG [RS_FLUSH_OPERATIONS-regionserver/037c58a19ddb:0-2 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731751192181.697cd6a363e514bff0d4ec954fc4034b. 2024-11-16T09:59:55,450 DEBUG [RS_FLUSH_OPERATIONS-regionserver/037c58a19ddb:0-2 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731751192181.697cd6a363e514bff0d4ec954fc4034b. as already flushing 2024-11-16T09:59:55,450 DEBUG [RS_FLUSH_OPERATIONS-regionserver/037c58a19ddb:0-2 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731751192181.697cd6a363e514bff0d4ec954fc4034b. 2024-11-16T09:59:55,450 ERROR [RS_FLUSH_OPERATIONS-regionserver/037c58a19ddb:0-2 {event_type=RS_FLUSH_REGIONS, pid=120}] handler.RSProcedureHandler(58): pid=120 java.io.IOException: Unable to complete flush {ENCODED => 697cd6a363e514bff0d4ec954fc4034b, NAME => 'TestAcidGuarantees,,1731751192181.697cd6a363e514bff0d4ec954fc4034b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T09:59:55,450 DEBUG [RS_FLUSH_OPERATIONS-regionserver/037c58a19ddb:0-2 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=120 java.io.IOException: Unable to complete flush {ENCODED => 697cd6a363e514bff0d4ec954fc4034b, NAME => 'TestAcidGuarantees,,1731751192181.697cd6a363e514bff0d4ec954fc4034b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T09:59:55,450 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37877 {}] master.HMaster(4177): Remote procedure failed, pid=120 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 697cd6a363e514bff0d4ec954fc4034b, NAME => 'TestAcidGuarantees,,1731751192181.697cd6a363e514bff0d4ec954fc4034b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 697cd6a363e514bff0d4ec954fc4034b, NAME => 'TestAcidGuarantees,,1731751192181.697cd6a363e514bff0d4ec954fc4034b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T09:59:55,513 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37877 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=119 2024-11-16T09:59:55,602 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=37267 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=120 2024-11-16T09:59:55,602 DEBUG [RS_FLUSH_OPERATIONS-regionserver/037c58a19ddb:0-0 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731751192181.697cd6a363e514bff0d4ec954fc4034b. 2024-11-16T09:59:55,603 DEBUG [RS_FLUSH_OPERATIONS-regionserver/037c58a19ddb:0-0 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731751192181.697cd6a363e514bff0d4ec954fc4034b. as already flushing 2024-11-16T09:59:55,603 DEBUG [RS_FLUSH_OPERATIONS-regionserver/037c58a19ddb:0-0 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731751192181.697cd6a363e514bff0d4ec954fc4034b. 2024-11-16T09:59:55,603 ERROR [RS_FLUSH_OPERATIONS-regionserver/037c58a19ddb:0-0 {event_type=RS_FLUSH_REGIONS, pid=120}] handler.RSProcedureHandler(58): pid=120 java.io.IOException: Unable to complete flush {ENCODED => 697cd6a363e514bff0d4ec954fc4034b, NAME => 'TestAcidGuarantees,,1731751192181.697cd6a363e514bff0d4ec954fc4034b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T09:59:55,603 DEBUG [RS_FLUSH_OPERATIONS-regionserver/037c58a19ddb:0-0 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=120 java.io.IOException: Unable to complete flush {ENCODED => 697cd6a363e514bff0d4ec954fc4034b, NAME => 'TestAcidGuarantees,,1731751192181.697cd6a363e514bff0d4ec954fc4034b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T09:59:55,603 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37877 {}] master.HMaster(4177): Remote procedure failed, pid=120 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 697cd6a363e514bff0d4ec954fc4034b, NAME => 'TestAcidGuarantees,,1731751192181.697cd6a363e514bff0d4ec954fc4034b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 697cd6a363e514bff0d4ec954fc4034b, NAME => 'TestAcidGuarantees,,1731751192181.697cd6a363e514bff0d4ec954fc4034b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T09:59:55,636 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=697cd6a363e514bff0d4ec954fc4034b, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T09:59:55,637 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36086 deadline: 1731751255633, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=697cd6a363e514bff0d4ec954fc4034b, server=037c58a19ddb,37267,1731751077844 2024-11-16T09:59:55,637 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731751192181.697cd6a363e514bff0d4ec954fc4034b., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 , the old value is region=TestAcidGuarantees,,1731751192181.697cd6a363e514bff0d4ec954fc4034b., hostname=037c58a19ddb,37267,1731751077844, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=697cd6a363e514bff0d4ec954fc4034b, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:59:55,637 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731751192181.697cd6a363e514bff0d4ec954fc4034b., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=697cd6a363e514bff0d4ec954fc4034b, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:59:55,637 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731751192181.697cd6a363e514bff0d4ec954fc4034b., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 because the exception is null or not the one we care about 2024-11-16T09:59:55,645 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=697cd6a363e514bff0d4ec954fc4034b, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T09:59:55,645 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36058 deadline: 1731751255643, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=697cd6a363e514bff0d4ec954fc4034b, server=037c58a19ddb,37267,1731751077844 2024-11-16T09:59:55,646 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731751192181.697cd6a363e514bff0d4ec954fc4034b., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 , the old value is region=TestAcidGuarantees,,1731751192181.697cd6a363e514bff0d4ec954fc4034b., hostname=037c58a19ddb,37267,1731751077844, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=697cd6a363e514bff0d4ec954fc4034b, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:59:55,646 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731751192181.697cd6a363e514bff0d4ec954fc4034b., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=697cd6a363e514bff0d4ec954fc4034b, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:59:55,646 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731751192181.697cd6a363e514bff0d4ec954fc4034b., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 because the exception is null or not the one we care about 2024-11-16T09:59:55,660 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=697cd6a363e514bff0d4ec954fc4034b, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T09:59:55,660 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36012 deadline: 1731751255653, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=697cd6a363e514bff0d4ec954fc4034b, server=037c58a19ddb,37267,1731751077844 2024-11-16T09:59:55,660 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=697cd6a363e514bff0d4ec954fc4034b, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T09:59:55,660 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36046 deadline: 1731751255653, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=697cd6a363e514bff0d4ec954fc4034b, server=037c58a19ddb,37267,1731751077844 2024-11-16T09:59:55,660 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731751192181.697cd6a363e514bff0d4ec954fc4034b., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 , the old value is region=TestAcidGuarantees,,1731751192181.697cd6a363e514bff0d4ec954fc4034b., hostname=037c58a19ddb,37267,1731751077844, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=697cd6a363e514bff0d4ec954fc4034b, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:59:55,661 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731751192181.697cd6a363e514bff0d4ec954fc4034b., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=697cd6a363e514bff0d4ec954fc4034b, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:59:55,661 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731751192181.697cd6a363e514bff0d4ec954fc4034b., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 because the exception is null or not the one we care about 2024-11-16T09:59:55,661 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=697cd6a363e514bff0d4ec954fc4034b, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T09:59:55,661 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36022 deadline: 1731751255653, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=697cd6a363e514bff0d4ec954fc4034b, server=037c58a19ddb,37267,1731751077844 2024-11-16T09:59:55,661 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731751192181.697cd6a363e514bff0d4ec954fc4034b., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 , the old value is region=TestAcidGuarantees,,1731751192181.697cd6a363e514bff0d4ec954fc4034b., hostname=037c58a19ddb,37267,1731751077844, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=697cd6a363e514bff0d4ec954fc4034b, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:59:55,661 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731751192181.697cd6a363e514bff0d4ec954fc4034b., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=697cd6a363e514bff0d4ec954fc4034b, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:59:55,661 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731751192181.697cd6a363e514bff0d4ec954fc4034b., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 because the exception is null or not the one we care about 2024-11-16T09:59:55,661 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731751192181.697cd6a363e514bff0d4ec954fc4034b., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 , the old value is region=TestAcidGuarantees,,1731751192181.697cd6a363e514bff0d4ec954fc4034b., hostname=037c58a19ddb,37267,1731751077844, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=697cd6a363e514bff0d4ec954fc4034b, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:59:55,661 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731751192181.697cd6a363e514bff0d4ec954fc4034b., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=697cd6a363e514bff0d4ec954fc4034b, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:59:55,661 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731751192181.697cd6a363e514bff0d4ec954fc4034b., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 because the exception is null or not the one we care about 2024-11-16T09:59:55,688 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=12 (bloomFilter=true), to=hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/697cd6a363e514bff0d4ec954fc4034b/.tmp/C/a5314ebc7f464faba37fa36b02402253 2024-11-16T09:59:55,692 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/697cd6a363e514bff0d4ec954fc4034b/.tmp/A/656d8bd3c1f740f590bef993761e4820 as hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/697cd6a363e514bff0d4ec954fc4034b/A/656d8bd3c1f740f590bef993761e4820 2024-11-16T09:59:55,695 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/697cd6a363e514bff0d4ec954fc4034b/A/656d8bd3c1f740f590bef993761e4820, entries=150, sequenceid=12, filesize=11.7 K 2024-11-16T09:59:55,695 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/697cd6a363e514bff0d4ec954fc4034b/.tmp/B/b879c013fa994e78b1fe7c6327ecd121 as hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/697cd6a363e514bff0d4ec954fc4034b/B/b879c013fa994e78b1fe7c6327ecd121 2024-11-16T09:59:55,699 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/697cd6a363e514bff0d4ec954fc4034b/B/b879c013fa994e78b1fe7c6327ecd121, entries=150, sequenceid=12, filesize=11.7 K 2024-11-16T09:59:55,699 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/697cd6a363e514bff0d4ec954fc4034b/.tmp/C/a5314ebc7f464faba37fa36b02402253 as hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/697cd6a363e514bff0d4ec954fc4034b/C/a5314ebc7f464faba37fa36b02402253 2024-11-16T09:59:55,702 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/697cd6a363e514bff0d4ec954fc4034b/C/a5314ebc7f464faba37fa36b02402253, entries=150, sequenceid=12, filesize=11.7 K 2024-11-16T09:59:55,703 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~141.33 KB/144720, currentSize=147.60 KB/151140 for 697cd6a363e514bff0d4ec954fc4034b in 1313ms, sequenceid=12, compaction requested=false 2024-11-16T09:59:55,703 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 697cd6a363e514bff0d4ec954fc4034b: 2024-11-16T09:59:55,755 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=37267 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=120 2024-11-16T09:59:55,755 DEBUG [RS_FLUSH_OPERATIONS-regionserver/037c58a19ddb:0-1 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731751192181.697cd6a363e514bff0d4ec954fc4034b. 2024-11-16T09:59:55,756 INFO [RS_FLUSH_OPERATIONS-regionserver/037c58a19ddb:0-1 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.HRegion(2902): Flushing 697cd6a363e514bff0d4ec954fc4034b 3/3 column families, dataSize=147.60 KB heapSize=387.47 KB 2024-11-16T09:59:55,761 DEBUG [RS_FLUSH_OPERATIONS-regionserver/037c58a19ddb:0-1 {event_type=RS_FLUSH_REGIONS, pid=120}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/697cd6a363e514bff0d4ec954fc4034b/.tmp/A/aedcfb03de7247b49de87f00d96a1a32 is 50, key is test_row_0/A:col10/1731751194485/Put/seqid=0 2024-11-16T09:59:55,765 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37591 is added to blk_1073742271_1447 (size=12001) 2024-11-16T09:59:55,865 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-11-16T09:59:56,165 INFO [RS_FLUSH_OPERATIONS-regionserver/037c58a19ddb:0-1 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=37 (bloomFilter=true), to=hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/697cd6a363e514bff0d4ec954fc4034b/.tmp/A/aedcfb03de7247b49de87f00d96a1a32 2024-11-16T09:59:56,172 DEBUG [RS_FLUSH_OPERATIONS-regionserver/037c58a19ddb:0-1 {event_type=RS_FLUSH_REGIONS, pid=120}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/697cd6a363e514bff0d4ec954fc4034b/.tmp/B/45b14ada2cc046159725de6299e3026e is 50, key is test_row_0/B:col10/1731751194485/Put/seqid=0 2024-11-16T09:59:56,176 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37591 is added to blk_1073742272_1448 (size=12001) 2024-11-16T09:59:56,179 INFO [RS_FLUSH_OPERATIONS-regionserver/037c58a19ddb:0-1 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=37 (bloomFilter=true), to=hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/697cd6a363e514bff0d4ec954fc4034b/.tmp/B/45b14ada2cc046159725de6299e3026e 2024-11-16T09:59:56,187 DEBUG [RS_FLUSH_OPERATIONS-regionserver/037c58a19ddb:0-1 {event_type=RS_FLUSH_REGIONS, pid=120}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/697cd6a363e514bff0d4ec954fc4034b/.tmp/C/1180197cb1f34930b834b28b614575b2 is 50, key is test_row_0/C:col10/1731751194485/Put/seqid=0 2024-11-16T09:59:56,190 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37591 is added to blk_1073742273_1449 (size=12001) 2024-11-16T09:59:56,192 INFO [RS_FLUSH_OPERATIONS-regionserver/037c58a19ddb:0-1 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=37 (bloomFilter=true), to=hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/697cd6a363e514bff0d4ec954fc4034b/.tmp/C/1180197cb1f34930b834b28b614575b2 2024-11-16T09:59:56,196 DEBUG [RS_FLUSH_OPERATIONS-regionserver/037c58a19ddb:0-1 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/697cd6a363e514bff0d4ec954fc4034b/.tmp/A/aedcfb03de7247b49de87f00d96a1a32 as hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/697cd6a363e514bff0d4ec954fc4034b/A/aedcfb03de7247b49de87f00d96a1a32 2024-11-16T09:59:56,199 INFO [RS_FLUSH_OPERATIONS-regionserver/037c58a19ddb:0-1 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/697cd6a363e514bff0d4ec954fc4034b/A/aedcfb03de7247b49de87f00d96a1a32, entries=150, sequenceid=37, filesize=11.7 K 2024-11-16T09:59:56,200 DEBUG [RS_FLUSH_OPERATIONS-regionserver/037c58a19ddb:0-1 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/697cd6a363e514bff0d4ec954fc4034b/.tmp/B/45b14ada2cc046159725de6299e3026e as hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/697cd6a363e514bff0d4ec954fc4034b/B/45b14ada2cc046159725de6299e3026e 2024-11-16T09:59:56,204 INFO [RS_FLUSH_OPERATIONS-regionserver/037c58a19ddb:0-1 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/697cd6a363e514bff0d4ec954fc4034b/B/45b14ada2cc046159725de6299e3026e, entries=150, sequenceid=37, filesize=11.7 K 2024-11-16T09:59:56,204 DEBUG [RS_FLUSH_OPERATIONS-regionserver/037c58a19ddb:0-1 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/697cd6a363e514bff0d4ec954fc4034b/.tmp/C/1180197cb1f34930b834b28b614575b2 as hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/697cd6a363e514bff0d4ec954fc4034b/C/1180197cb1f34930b834b28b614575b2 2024-11-16T09:59:56,208 INFO [RS_FLUSH_OPERATIONS-regionserver/037c58a19ddb:0-1 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/697cd6a363e514bff0d4ec954fc4034b/C/1180197cb1f34930b834b28b614575b2, entries=150, sequenceid=37, filesize=11.7 K 2024-11-16T09:59:56,209 INFO [RS_FLUSH_OPERATIONS-regionserver/037c58a19ddb:0-1 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.HRegion(3140): Finished flush of dataSize ~147.60 KB/151140, heapSize ~387.42 KB/396720, currentSize=0 B/0 for 697cd6a363e514bff0d4ec954fc4034b in 452ms, sequenceid=37, compaction requested=false 2024-11-16T09:59:56,209 DEBUG [RS_FLUSH_OPERATIONS-regionserver/037c58a19ddb:0-1 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.HRegion(2603): Flush status journal for 697cd6a363e514bff0d4ec954fc4034b: 2024-11-16T09:59:56,209 DEBUG [RS_FLUSH_OPERATIONS-regionserver/037c58a19ddb:0-1 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731751192181.697cd6a363e514bff0d4ec954fc4034b. 2024-11-16T09:59:56,209 DEBUG [RS_FLUSH_OPERATIONS-regionserver/037c58a19ddb:0-1 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=120 2024-11-16T09:59:56,209 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37877 {}] master.HMaster(4169): Remote procedure done, pid=120 2024-11-16T09:59:56,213 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=120, resume processing ppid=119 2024-11-16T09:59:56,213 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=120, ppid=119, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.8310 sec 2024-11-16T09:59:56,215 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=119, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=119, table=TestAcidGuarantees in 1.8400 sec 2024-11-16T09:59:56,236 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtil$FsDatasetAsyncDiskServiceFixer(597): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details. 2024-11-16T09:59:56,523 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37877 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=119 2024-11-16T09:59:56,523 INFO [RPCClient-NioEventLoopGroup-4-10 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-16T09:59:56,524 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37877 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-16T09:59:56,525 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37877 {}] procedure2.ProcedureExecutor(1139): Stored pid=121, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=121, table=TestAcidGuarantees 2024-11-16T09:59:56,525 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37877 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=121 2024-11-16T09:59:56,526 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=121, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=121, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-16T09:59:56,527 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=121, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=121, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-16T09:59:56,527 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=122, ppid=121, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-16T09:59:56,633 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37877 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=121 2024-11-16T09:59:56,664 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] regionserver.HRegion(8855): Flush requested on 697cd6a363e514bff0d4ec954fc4034b 2024-11-16T09:59:56,664 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-16T09:59:56,665 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 697cd6a363e514bff0d4ec954fc4034b 3/3 column families, dataSize=53.67 KB heapSize=141.38 KB 2024-11-16T09:59:56,669 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/697cd6a363e514bff0d4ec954fc4034b/.tmp/A/38f40b75e4b343319edd57d2aca67982 is 50, key is test_row_0/A:col10/1731751196662/Put/seqid=0 2024-11-16T09:59:56,679 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=37267 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=122 2024-11-16T09:59:56,679 DEBUG [RS_FLUSH_OPERATIONS-regionserver/037c58a19ddb:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731751192181.697cd6a363e514bff0d4ec954fc4034b. 2024-11-16T09:59:56,680 DEBUG [RS_FLUSH_OPERATIONS-regionserver/037c58a19ddb:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731751192181.697cd6a363e514bff0d4ec954fc4034b. as already flushing 2024-11-16T09:59:56,680 DEBUG [RS_FLUSH_OPERATIONS-regionserver/037c58a19ddb:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731751192181.697cd6a363e514bff0d4ec954fc4034b. 2024-11-16T09:59:56,680 ERROR [RS_FLUSH_OPERATIONS-regionserver/037c58a19ddb:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] handler.RSProcedureHandler(58): pid=122 java.io.IOException: Unable to complete flush {ENCODED => 697cd6a363e514bff0d4ec954fc4034b, NAME => 'TestAcidGuarantees,,1731751192181.697cd6a363e514bff0d4ec954fc4034b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T09:59:56,680 DEBUG [RS_FLUSH_OPERATIONS-regionserver/037c58a19ddb:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=122 java.io.IOException: Unable to complete flush {ENCODED => 697cd6a363e514bff0d4ec954fc4034b, NAME => 'TestAcidGuarantees,,1731751192181.697cd6a363e514bff0d4ec954fc4034b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T09:59:56,681 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37877 {}] master.HMaster(4177): Remote procedure failed, pid=122 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 697cd6a363e514bff0d4ec954fc4034b, NAME => 'TestAcidGuarantees,,1731751192181.697cd6a363e514bff0d4ec954fc4034b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 697cd6a363e514bff0d4ec954fc4034b, NAME => 'TestAcidGuarantees,,1731751192181.697cd6a363e514bff0d4ec954fc4034b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T09:59:56,685 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37591 is added to blk_1073742274_1450 (size=14341) 2024-11-16T09:59:56,713 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=697cd6a363e514bff0d4ec954fc4034b, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T09:59:56,714 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36086 deadline: 1731751256701, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=697cd6a363e514bff0d4ec954fc4034b, server=037c58a19ddb,37267,1731751077844 2024-11-16T09:59:56,714 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=697cd6a363e514bff0d4ec954fc4034b, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T09:59:56,714 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731751192181.697cd6a363e514bff0d4ec954fc4034b., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 , the old value is region=TestAcidGuarantees,,1731751192181.697cd6a363e514bff0d4ec954fc4034b., hostname=037c58a19ddb,37267,1731751077844, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=697cd6a363e514bff0d4ec954fc4034b, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:59:56,714 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731751192181.697cd6a363e514bff0d4ec954fc4034b., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=697cd6a363e514bff0d4ec954fc4034b, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:59:56,714 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731751192181.697cd6a363e514bff0d4ec954fc4034b., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 because the exception is null or not the one we care about 2024-11-16T09:59:56,714 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36012 deadline: 1731751256710, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=697cd6a363e514bff0d4ec954fc4034b, server=037c58a19ddb,37267,1731751077844 2024-11-16T09:59:56,714 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731751192181.697cd6a363e514bff0d4ec954fc4034b., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 , the old value is region=TestAcidGuarantees,,1731751192181.697cd6a363e514bff0d4ec954fc4034b., hostname=037c58a19ddb,37267,1731751077844, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=697cd6a363e514bff0d4ec954fc4034b, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:59:56,714 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731751192181.697cd6a363e514bff0d4ec954fc4034b., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=697cd6a363e514bff0d4ec954fc4034b, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:59:56,714 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731751192181.697cd6a363e514bff0d4ec954fc4034b., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 because the exception is null or not the one we care about 2024-11-16T09:59:56,720 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=697cd6a363e514bff0d4ec954fc4034b, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T09:59:56,720 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36022 deadline: 1731751256711, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=697cd6a363e514bff0d4ec954fc4034b, server=037c58a19ddb,37267,1731751077844 2024-11-16T09:59:56,721 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=697cd6a363e514bff0d4ec954fc4034b, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T09:59:56,721 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731751192181.697cd6a363e514bff0d4ec954fc4034b., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 , the old value is region=TestAcidGuarantees,,1731751192181.697cd6a363e514bff0d4ec954fc4034b., hostname=037c58a19ddb,37267,1731751077844, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=697cd6a363e514bff0d4ec954fc4034b, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:59:56,721 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36058 deadline: 1731751256712, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=697cd6a363e514bff0d4ec954fc4034b, server=037c58a19ddb,37267,1731751077844 2024-11-16T09:59:56,721 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731751192181.697cd6a363e514bff0d4ec954fc4034b., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=697cd6a363e514bff0d4ec954fc4034b, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:59:56,721 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731751192181.697cd6a363e514bff0d4ec954fc4034b., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 because the exception is null or not the one we care about 2024-11-16T09:59:56,721 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731751192181.697cd6a363e514bff0d4ec954fc4034b., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 , the old value is region=TestAcidGuarantees,,1731751192181.697cd6a363e514bff0d4ec954fc4034b., hostname=037c58a19ddb,37267,1731751077844, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=697cd6a363e514bff0d4ec954fc4034b, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:59:56,721 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731751192181.697cd6a363e514bff0d4ec954fc4034b., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=697cd6a363e514bff0d4ec954fc4034b, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:59:56,721 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731751192181.697cd6a363e514bff0d4ec954fc4034b., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 because the exception is null or not the one we care about 2024-11-16T09:59:56,724 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=697cd6a363e514bff0d4ec954fc4034b, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T09:59:56,724 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36046 deadline: 1731751256714, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=697cd6a363e514bff0d4ec954fc4034b, server=037c58a19ddb,37267,1731751077844 2024-11-16T09:59:56,725 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731751192181.697cd6a363e514bff0d4ec954fc4034b., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 , the old value is region=TestAcidGuarantees,,1731751192181.697cd6a363e514bff0d4ec954fc4034b., hostname=037c58a19ddb,37267,1731751077844, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=697cd6a363e514bff0d4ec954fc4034b, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:59:56,725 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731751192181.697cd6a363e514bff0d4ec954fc4034b., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=697cd6a363e514bff0d4ec954fc4034b, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:59:56,725 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731751192181.697cd6a363e514bff0d4ec954fc4034b., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 because the exception is null or not the one we care about 2024-11-16T09:59:56,830 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=697cd6a363e514bff0d4ec954fc4034b, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T09:59:56,830 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36086 deadline: 1731751256823, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=697cd6a363e514bff0d4ec954fc4034b, server=037c58a19ddb,37267,1731751077844 2024-11-16T09:59:56,830 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=697cd6a363e514bff0d4ec954fc4034b, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T09:59:56,830 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731751192181.697cd6a363e514bff0d4ec954fc4034b., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 , the old value is region=TestAcidGuarantees,,1731751192181.697cd6a363e514bff0d4ec954fc4034b., hostname=037c58a19ddb,37267,1731751077844, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=697cd6a363e514bff0d4ec954fc4034b, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:59:56,830 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731751192181.697cd6a363e514bff0d4ec954fc4034b., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=697cd6a363e514bff0d4ec954fc4034b, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:59:56,830 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36012 deadline: 1731751256823, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=697cd6a363e514bff0d4ec954fc4034b, server=037c58a19ddb,37267,1731751077844 2024-11-16T09:59:56,830 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731751192181.697cd6a363e514bff0d4ec954fc4034b., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 because the exception is null or not the one we care about 2024-11-16T09:59:56,831 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=697cd6a363e514bff0d4ec954fc4034b, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T09:59:56,831 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731751192181.697cd6a363e514bff0d4ec954fc4034b., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 , the old value is region=TestAcidGuarantees,,1731751192181.697cd6a363e514bff0d4ec954fc4034b., hostname=037c58a19ddb,37267,1731751077844, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=697cd6a363e514bff0d4ec954fc4034b, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:59:56,831 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731751192181.697cd6a363e514bff0d4ec954fc4034b., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=697cd6a363e514bff0d4ec954fc4034b, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:59:56,831 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731751192181.697cd6a363e514bff0d4ec954fc4034b., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 because the exception is null or not the one we care about 2024-11-16T09:59:56,831 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36022 deadline: 1731751256823, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=697cd6a363e514bff0d4ec954fc4034b, server=037c58a19ddb,37267,1731751077844 2024-11-16T09:59:56,831 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731751192181.697cd6a363e514bff0d4ec954fc4034b., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 , the old value is region=TestAcidGuarantees,,1731751192181.697cd6a363e514bff0d4ec954fc4034b., hostname=037c58a19ddb,37267,1731751077844, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=697cd6a363e514bff0d4ec954fc4034b, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:59:56,831 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731751192181.697cd6a363e514bff0d4ec954fc4034b., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=697cd6a363e514bff0d4ec954fc4034b, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:59:56,831 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731751192181.697cd6a363e514bff0d4ec954fc4034b., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 because the exception is null or not the one we care about 2024-11-16T09:59:56,833 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=37267 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=122 2024-11-16T09:59:56,833 DEBUG [RS_FLUSH_OPERATIONS-regionserver/037c58a19ddb:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731751192181.697cd6a363e514bff0d4ec954fc4034b. 2024-11-16T09:59:56,833 DEBUG [RS_FLUSH_OPERATIONS-regionserver/037c58a19ddb:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731751192181.697cd6a363e514bff0d4ec954fc4034b. as already flushing 2024-11-16T09:59:56,833 DEBUG [RS_FLUSH_OPERATIONS-regionserver/037c58a19ddb:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731751192181.697cd6a363e514bff0d4ec954fc4034b. 2024-11-16T09:59:56,833 ERROR [RS_FLUSH_OPERATIONS-regionserver/037c58a19ddb:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] handler.RSProcedureHandler(58): pid=122 java.io.IOException: Unable to complete flush {ENCODED => 697cd6a363e514bff0d4ec954fc4034b, NAME => 'TestAcidGuarantees,,1731751192181.697cd6a363e514bff0d4ec954fc4034b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T09:59:56,833 DEBUG [RS_FLUSH_OPERATIONS-regionserver/037c58a19ddb:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=122 java.io.IOException: Unable to complete flush {ENCODED => 697cd6a363e514bff0d4ec954fc4034b, NAME => 'TestAcidGuarantees,,1731751192181.697cd6a363e514bff0d4ec954fc4034b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T09:59:56,834 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37877 {}] master.HMaster(4177): Remote procedure failed, pid=122 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 697cd6a363e514bff0d4ec954fc4034b, NAME => 'TestAcidGuarantees,,1731751192181.697cd6a363e514bff0d4ec954fc4034b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 697cd6a363e514bff0d4ec954fc4034b, NAME => 'TestAcidGuarantees,,1731751192181.697cd6a363e514bff0d4ec954fc4034b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T09:59:56,840 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=697cd6a363e514bff0d4ec954fc4034b, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T09:59:56,841 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36046 deadline: 1731751256833, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=697cd6a363e514bff0d4ec954fc4034b, server=037c58a19ddb,37267,1731751077844 2024-11-16T09:59:56,841 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731751192181.697cd6a363e514bff0d4ec954fc4034b., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 , the old value is region=TestAcidGuarantees,,1731751192181.697cd6a363e514bff0d4ec954fc4034b., hostname=037c58a19ddb,37267,1731751077844, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=697cd6a363e514bff0d4ec954fc4034b, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:59:56,841 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=697cd6a363e514bff0d4ec954fc4034b, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T09:59:56,841 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731751192181.697cd6a363e514bff0d4ec954fc4034b., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=697cd6a363e514bff0d4ec954fc4034b, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:59:56,841 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731751192181.697cd6a363e514bff0d4ec954fc4034b., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 because the exception is null or not the one we care about 2024-11-16T09:59:56,841 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36058 deadline: 1731751256833, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=697cd6a363e514bff0d4ec954fc4034b, server=037c58a19ddb,37267,1731751077844 2024-11-16T09:59:56,841 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731751192181.697cd6a363e514bff0d4ec954fc4034b., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 , the old value is region=TestAcidGuarantees,,1731751192181.697cd6a363e514bff0d4ec954fc4034b., hostname=037c58a19ddb,37267,1731751077844, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=697cd6a363e514bff0d4ec954fc4034b, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:59:56,841 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731751192181.697cd6a363e514bff0d4ec954fc4034b., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=697cd6a363e514bff0d4ec954fc4034b, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:59:56,841 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731751192181.697cd6a363e514bff0d4ec954fc4034b., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 because the exception is null or not the one we care about 2024-11-16T09:59:56,842 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37877 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=121 2024-11-16T09:59:56,985 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=37267 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=122 2024-11-16T09:59:56,986 DEBUG [RS_FLUSH_OPERATIONS-regionserver/037c58a19ddb:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731751192181.697cd6a363e514bff0d4ec954fc4034b. 2024-11-16T09:59:56,986 DEBUG [RS_FLUSH_OPERATIONS-regionserver/037c58a19ddb:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731751192181.697cd6a363e514bff0d4ec954fc4034b. as already flushing 2024-11-16T09:59:56,986 DEBUG [RS_FLUSH_OPERATIONS-regionserver/037c58a19ddb:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731751192181.697cd6a363e514bff0d4ec954fc4034b. 2024-11-16T09:59:56,986 ERROR [RS_FLUSH_OPERATIONS-regionserver/037c58a19ddb:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] handler.RSProcedureHandler(58): pid=122 java.io.IOException: Unable to complete flush {ENCODED => 697cd6a363e514bff0d4ec954fc4034b, NAME => 'TestAcidGuarantees,,1731751192181.697cd6a363e514bff0d4ec954fc4034b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T09:59:56,986 DEBUG [RS_FLUSH_OPERATIONS-regionserver/037c58a19ddb:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=122 java.io.IOException: Unable to complete flush {ENCODED => 697cd6a363e514bff0d4ec954fc4034b, NAME => 'TestAcidGuarantees,,1731751192181.697cd6a363e514bff0d4ec954fc4034b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T09:59:56,987 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37877 {}] master.HMaster(4177): Remote procedure failed, pid=122 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 697cd6a363e514bff0d4ec954fc4034b, NAME => 'TestAcidGuarantees,,1731751192181.697cd6a363e514bff0d4ec954fc4034b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 697cd6a363e514bff0d4ec954fc4034b, NAME => 'TestAcidGuarantees,,1731751192181.697cd6a363e514bff0d4ec954fc4034b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T09:59:57,041 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=697cd6a363e514bff0d4ec954fc4034b, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T09:59:57,041 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36012 deadline: 1731751257033, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=697cd6a363e514bff0d4ec954fc4034b, server=037c58a19ddb,37267,1731751077844 2024-11-16T09:59:57,041 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731751192181.697cd6a363e514bff0d4ec954fc4034b., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 , the old value is region=TestAcidGuarantees,,1731751192181.697cd6a363e514bff0d4ec954fc4034b., hostname=037c58a19ddb,37267,1731751077844, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=697cd6a363e514bff0d4ec954fc4034b, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:59:57,041 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731751192181.697cd6a363e514bff0d4ec954fc4034b., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=697cd6a363e514bff0d4ec954fc4034b, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:59:57,041 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731751192181.697cd6a363e514bff0d4ec954fc4034b., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 because the exception is null or not the one we care about 2024-11-16T09:59:57,051 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=697cd6a363e514bff0d4ec954fc4034b, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T09:59:57,051 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36086 deadline: 1731751257043, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=697cd6a363e514bff0d4ec954fc4034b, server=037c58a19ddb,37267,1731751077844 2024-11-16T09:59:57,051 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731751192181.697cd6a363e514bff0d4ec954fc4034b., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 , the old value is region=TestAcidGuarantees,,1731751192181.697cd6a363e514bff0d4ec954fc4034b., hostname=037c58a19ddb,37267,1731751077844, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=697cd6a363e514bff0d4ec954fc4034b, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:59:57,051 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=697cd6a363e514bff0d4ec954fc4034b, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T09:59:57,051 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731751192181.697cd6a363e514bff0d4ec954fc4034b., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=697cd6a363e514bff0d4ec954fc4034b, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:59:57,051 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731751192181.697cd6a363e514bff0d4ec954fc4034b., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 because the exception is null or not the one we care about 2024-11-16T09:59:57,051 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36022 deadline: 1731751257043, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=697cd6a363e514bff0d4ec954fc4034b, server=037c58a19ddb,37267,1731751077844 2024-11-16T09:59:57,051 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731751192181.697cd6a363e514bff0d4ec954fc4034b., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 , the old value is region=TestAcidGuarantees,,1731751192181.697cd6a363e514bff0d4ec954fc4034b., hostname=037c58a19ddb,37267,1731751077844, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=697cd6a363e514bff0d4ec954fc4034b, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:59:57,051 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=697cd6a363e514bff0d4ec954fc4034b, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T09:59:57,051 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731751192181.697cd6a363e514bff0d4ec954fc4034b., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=697cd6a363e514bff0d4ec954fc4034b, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:59:57,051 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731751192181.697cd6a363e514bff0d4ec954fc4034b., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 because the exception is null or not the one we care about 2024-11-16T09:59:57,051 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36058 deadline: 1731751257043, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=697cd6a363e514bff0d4ec954fc4034b, server=037c58a19ddb,37267,1731751077844 2024-11-16T09:59:57,052 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731751192181.697cd6a363e514bff0d4ec954fc4034b., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 , the old value is region=TestAcidGuarantees,,1731751192181.697cd6a363e514bff0d4ec954fc4034b., hostname=037c58a19ddb,37267,1731751077844, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=697cd6a363e514bff0d4ec954fc4034b, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:59:57,052 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731751192181.697cd6a363e514bff0d4ec954fc4034b., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=697cd6a363e514bff0d4ec954fc4034b, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:59:57,052 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731751192181.697cd6a363e514bff0d4ec954fc4034b., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 because the exception is null or not the one we care about 2024-11-16T09:59:57,053 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=697cd6a363e514bff0d4ec954fc4034b, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T09:59:57,053 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36046 deadline: 1731751257052, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=697cd6a363e514bff0d4ec954fc4034b, server=037c58a19ddb,37267,1731751077844 2024-11-16T09:59:57,053 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731751192181.697cd6a363e514bff0d4ec954fc4034b., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 , the old value is region=TestAcidGuarantees,,1731751192181.697cd6a363e514bff0d4ec954fc4034b., hostname=037c58a19ddb,37267,1731751077844, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=697cd6a363e514bff0d4ec954fc4034b, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:59:57,053 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731751192181.697cd6a363e514bff0d4ec954fc4034b., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=697cd6a363e514bff0d4ec954fc4034b, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:59:57,054 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731751192181.697cd6a363e514bff0d4ec954fc4034b., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 because the exception is null or not the one we care about 2024-11-16T09:59:57,086 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=48 (bloomFilter=true), to=hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/697cd6a363e514bff0d4ec954fc4034b/.tmp/A/38f40b75e4b343319edd57d2aca67982 2024-11-16T09:59:57,092 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/697cd6a363e514bff0d4ec954fc4034b/.tmp/B/19d2ff883d9a4216a9c2e142390c211d is 50, key is test_row_0/B:col10/1731751196662/Put/seqid=0 2024-11-16T09:59:57,096 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37591 is added to blk_1073742275_1451 (size=12001) 2024-11-16T09:59:57,138 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=37267 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=122 2024-11-16T09:59:57,139 DEBUG [RS_FLUSH_OPERATIONS-regionserver/037c58a19ddb:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731751192181.697cd6a363e514bff0d4ec954fc4034b. 2024-11-16T09:59:57,139 DEBUG [RS_FLUSH_OPERATIONS-regionserver/037c58a19ddb:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731751192181.697cd6a363e514bff0d4ec954fc4034b. as already flushing 2024-11-16T09:59:57,139 DEBUG [RS_FLUSH_OPERATIONS-regionserver/037c58a19ddb:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731751192181.697cd6a363e514bff0d4ec954fc4034b. 2024-11-16T09:59:57,139 ERROR [RS_FLUSH_OPERATIONS-regionserver/037c58a19ddb:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] handler.RSProcedureHandler(58): pid=122 java.io.IOException: Unable to complete flush {ENCODED => 697cd6a363e514bff0d4ec954fc4034b, NAME => 'TestAcidGuarantees,,1731751192181.697cd6a363e514bff0d4ec954fc4034b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T09:59:57,139 DEBUG [RS_FLUSH_OPERATIONS-regionserver/037c58a19ddb:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=122 java.io.IOException: Unable to complete flush {ENCODED => 697cd6a363e514bff0d4ec954fc4034b, NAME => 'TestAcidGuarantees,,1731751192181.697cd6a363e514bff0d4ec954fc4034b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T09:59:57,139 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37877 {}] master.HMaster(4177): Remote procedure failed, pid=122 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 697cd6a363e514bff0d4ec954fc4034b, NAME => 'TestAcidGuarantees,,1731751192181.697cd6a363e514bff0d4ec954fc4034b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 697cd6a363e514bff0d4ec954fc4034b, NAME => 'TestAcidGuarantees,,1731751192181.697cd6a363e514bff0d4ec954fc4034b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T09:59:57,152 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37877 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=121 2024-11-16T09:59:57,291 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=37267 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=122 2024-11-16T09:59:57,291 DEBUG [RS_FLUSH_OPERATIONS-regionserver/037c58a19ddb:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731751192181.697cd6a363e514bff0d4ec954fc4034b. 2024-11-16T09:59:57,291 DEBUG [RS_FLUSH_OPERATIONS-regionserver/037c58a19ddb:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731751192181.697cd6a363e514bff0d4ec954fc4034b. as already flushing 2024-11-16T09:59:57,291 DEBUG [RS_FLUSH_OPERATIONS-regionserver/037c58a19ddb:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731751192181.697cd6a363e514bff0d4ec954fc4034b. 2024-11-16T09:59:57,291 ERROR [RS_FLUSH_OPERATIONS-regionserver/037c58a19ddb:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] handler.RSProcedureHandler(58): pid=122 java.io.IOException: Unable to complete flush {ENCODED => 697cd6a363e514bff0d4ec954fc4034b, NAME => 'TestAcidGuarantees,,1731751192181.697cd6a363e514bff0d4ec954fc4034b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T09:59:57,292 DEBUG [RS_FLUSH_OPERATIONS-regionserver/037c58a19ddb:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=122 java.io.IOException: Unable to complete flush {ENCODED => 697cd6a363e514bff0d4ec954fc4034b, NAME => 'TestAcidGuarantees,,1731751192181.697cd6a363e514bff0d4ec954fc4034b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T09:59:57,292 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37877 {}] master.HMaster(4177): Remote procedure failed, pid=122 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 697cd6a363e514bff0d4ec954fc4034b, NAME => 'TestAcidGuarantees,,1731751192181.697cd6a363e514bff0d4ec954fc4034b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 697cd6a363e514bff0d4ec954fc4034b, NAME => 'TestAcidGuarantees,,1731751192181.697cd6a363e514bff0d4ec954fc4034b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T09:59:57,351 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=697cd6a363e514bff0d4ec954fc4034b, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T09:59:57,351 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36012 deadline: 1731751257343, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=697cd6a363e514bff0d4ec954fc4034b, server=037c58a19ddb,37267,1731751077844 2024-11-16T09:59:57,352 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731751192181.697cd6a363e514bff0d4ec954fc4034b., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 , the old value is region=TestAcidGuarantees,,1731751192181.697cd6a363e514bff0d4ec954fc4034b., hostname=037c58a19ddb,37267,1731751077844, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=697cd6a363e514bff0d4ec954fc4034b, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:59:57,352 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731751192181.697cd6a363e514bff0d4ec954fc4034b., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=697cd6a363e514bff0d4ec954fc4034b, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:59:57,352 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731751192181.697cd6a363e514bff0d4ec954fc4034b., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 because the exception is null or not the one we care about 2024-11-16T09:59:57,367 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=697cd6a363e514bff0d4ec954fc4034b, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T09:59:57,367 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36022 deadline: 1731751257363, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=697cd6a363e514bff0d4ec954fc4034b, server=037c58a19ddb,37267,1731751077844 2024-11-16T09:59:57,367 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=697cd6a363e514bff0d4ec954fc4034b, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T09:59:57,368 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731751192181.697cd6a363e514bff0d4ec954fc4034b., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 , the old value is region=TestAcidGuarantees,,1731751192181.697cd6a363e514bff0d4ec954fc4034b., hostname=037c58a19ddb,37267,1731751077844, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=697cd6a363e514bff0d4ec954fc4034b, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:59:57,368 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36086 deadline: 1731751257363, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=697cd6a363e514bff0d4ec954fc4034b, server=037c58a19ddb,37267,1731751077844 2024-11-16T09:59:57,368 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731751192181.697cd6a363e514bff0d4ec954fc4034b., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=697cd6a363e514bff0d4ec954fc4034b, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:59:57,368 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731751192181.697cd6a363e514bff0d4ec954fc4034b., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 because the exception is null or not the one we care about 2024-11-16T09:59:57,368 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=697cd6a363e514bff0d4ec954fc4034b, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T09:59:57,368 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36058 deadline: 1731751257363, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=697cd6a363e514bff0d4ec954fc4034b, server=037c58a19ddb,37267,1731751077844 2024-11-16T09:59:57,368 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731751192181.697cd6a363e514bff0d4ec954fc4034b., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 , the old value is region=TestAcidGuarantees,,1731751192181.697cd6a363e514bff0d4ec954fc4034b., hostname=037c58a19ddb,37267,1731751077844, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=697cd6a363e514bff0d4ec954fc4034b, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:59:57,368 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731751192181.697cd6a363e514bff0d4ec954fc4034b., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=697cd6a363e514bff0d4ec954fc4034b, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:59:57,368 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731751192181.697cd6a363e514bff0d4ec954fc4034b., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 because the exception is null or not the one we care about 2024-11-16T09:59:57,368 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731751192181.697cd6a363e514bff0d4ec954fc4034b., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 , the old value is region=TestAcidGuarantees,,1731751192181.697cd6a363e514bff0d4ec954fc4034b., hostname=037c58a19ddb,37267,1731751077844, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=697cd6a363e514bff0d4ec954fc4034b, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:59:57,368 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731751192181.697cd6a363e514bff0d4ec954fc4034b., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=697cd6a363e514bff0d4ec954fc4034b, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:59:57,368 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731751192181.697cd6a363e514bff0d4ec954fc4034b., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 because the exception is null or not the one we care about 2024-11-16T09:59:57,371 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=697cd6a363e514bff0d4ec954fc4034b, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T09:59:57,371 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36046 deadline: 1731751257363, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=697cd6a363e514bff0d4ec954fc4034b, server=037c58a19ddb,37267,1731751077844 2024-11-16T09:59:57,371 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731751192181.697cd6a363e514bff0d4ec954fc4034b., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 , the old value is region=TestAcidGuarantees,,1731751192181.697cd6a363e514bff0d4ec954fc4034b., hostname=037c58a19ddb,37267,1731751077844, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=697cd6a363e514bff0d4ec954fc4034b, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:59:57,371 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731751192181.697cd6a363e514bff0d4ec954fc4034b., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=697cd6a363e514bff0d4ec954fc4034b, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:59:57,371 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731751192181.697cd6a363e514bff0d4ec954fc4034b., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 because the exception is null or not the one we care about 2024-11-16T09:59:57,444 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=37267 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=122 2024-11-16T09:59:57,444 DEBUG [RS_FLUSH_OPERATIONS-regionserver/037c58a19ddb:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731751192181.697cd6a363e514bff0d4ec954fc4034b. 2024-11-16T09:59:57,444 DEBUG [RS_FLUSH_OPERATIONS-regionserver/037c58a19ddb:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731751192181.697cd6a363e514bff0d4ec954fc4034b. as already flushing 2024-11-16T09:59:57,444 DEBUG [RS_FLUSH_OPERATIONS-regionserver/037c58a19ddb:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731751192181.697cd6a363e514bff0d4ec954fc4034b. 2024-11-16T09:59:57,444 ERROR [RS_FLUSH_OPERATIONS-regionserver/037c58a19ddb:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] handler.RSProcedureHandler(58): pid=122 java.io.IOException: Unable to complete flush {ENCODED => 697cd6a363e514bff0d4ec954fc4034b, NAME => 'TestAcidGuarantees,,1731751192181.697cd6a363e514bff0d4ec954fc4034b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T09:59:57,444 DEBUG [RS_FLUSH_OPERATIONS-regionserver/037c58a19ddb:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=122 java.io.IOException: Unable to complete flush {ENCODED => 697cd6a363e514bff0d4ec954fc4034b, NAME => 'TestAcidGuarantees,,1731751192181.697cd6a363e514bff0d4ec954fc4034b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T09:59:57,445 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37877 {}] master.HMaster(4177): Remote procedure failed, pid=122 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 697cd6a363e514bff0d4ec954fc4034b, NAME => 'TestAcidGuarantees,,1731751192181.697cd6a363e514bff0d4ec954fc4034b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 697cd6a363e514bff0d4ec954fc4034b, NAME => 'TestAcidGuarantees,,1731751192181.697cd6a363e514bff0d4ec954fc4034b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T09:59:57,497 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=48 (bloomFilter=true), to=hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/697cd6a363e514bff0d4ec954fc4034b/.tmp/B/19d2ff883d9a4216a9c2e142390c211d 2024-11-16T09:59:57,503 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/697cd6a363e514bff0d4ec954fc4034b/.tmp/C/a7411494c98043698e28066a2f4b930b is 50, key is test_row_0/C:col10/1731751196662/Put/seqid=0 2024-11-16T09:59:57,506 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37591 is added to blk_1073742276_1452 (size=12001) 2024-11-16T09:59:57,597 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=37267 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=122 2024-11-16T09:59:57,597 DEBUG [RS_FLUSH_OPERATIONS-regionserver/037c58a19ddb:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731751192181.697cd6a363e514bff0d4ec954fc4034b. 2024-11-16T09:59:57,597 DEBUG [RS_FLUSH_OPERATIONS-regionserver/037c58a19ddb:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731751192181.697cd6a363e514bff0d4ec954fc4034b. as already flushing 2024-11-16T09:59:57,597 DEBUG [RS_FLUSH_OPERATIONS-regionserver/037c58a19ddb:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731751192181.697cd6a363e514bff0d4ec954fc4034b. 2024-11-16T09:59:57,597 ERROR [RS_FLUSH_OPERATIONS-regionserver/037c58a19ddb:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] handler.RSProcedureHandler(58): pid=122 java.io.IOException: Unable to complete flush {ENCODED => 697cd6a363e514bff0d4ec954fc4034b, NAME => 'TestAcidGuarantees,,1731751192181.697cd6a363e514bff0d4ec954fc4034b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T09:59:57,597 DEBUG [RS_FLUSH_OPERATIONS-regionserver/037c58a19ddb:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=122 java.io.IOException: Unable to complete flush {ENCODED => 697cd6a363e514bff0d4ec954fc4034b, NAME => 'TestAcidGuarantees,,1731751192181.697cd6a363e514bff0d4ec954fc4034b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T09:59:57,598 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37877 {}] master.HMaster(4177): Remote procedure failed, pid=122 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 697cd6a363e514bff0d4ec954fc4034b, NAME => 'TestAcidGuarantees,,1731751192181.697cd6a363e514bff0d4ec954fc4034b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 697cd6a363e514bff0d4ec954fc4034b, NAME => 'TestAcidGuarantees,,1731751192181.697cd6a363e514bff0d4ec954fc4034b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T09:59:57,663 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37877 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=121 2024-11-16T09:59:57,750 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=37267 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=122 2024-11-16T09:59:57,750 DEBUG [RS_FLUSH_OPERATIONS-regionserver/037c58a19ddb:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731751192181.697cd6a363e514bff0d4ec954fc4034b. 2024-11-16T09:59:57,750 DEBUG [RS_FLUSH_OPERATIONS-regionserver/037c58a19ddb:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731751192181.697cd6a363e514bff0d4ec954fc4034b. as already flushing 2024-11-16T09:59:57,750 DEBUG [RS_FLUSH_OPERATIONS-regionserver/037c58a19ddb:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731751192181.697cd6a363e514bff0d4ec954fc4034b. 2024-11-16T09:59:57,750 ERROR [RS_FLUSH_OPERATIONS-regionserver/037c58a19ddb:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] handler.RSProcedureHandler(58): pid=122 java.io.IOException: Unable to complete flush {ENCODED => 697cd6a363e514bff0d4ec954fc4034b, NAME => 'TestAcidGuarantees,,1731751192181.697cd6a363e514bff0d4ec954fc4034b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T09:59:57,751 DEBUG [RS_FLUSH_OPERATIONS-regionserver/037c58a19ddb:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=122 java.io.IOException: Unable to complete flush {ENCODED => 697cd6a363e514bff0d4ec954fc4034b, NAME => 'TestAcidGuarantees,,1731751192181.697cd6a363e514bff0d4ec954fc4034b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T09:59:57,751 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37877 {}] master.HMaster(4177): Remote procedure failed, pid=122 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 697cd6a363e514bff0d4ec954fc4034b, NAME => 'TestAcidGuarantees,,1731751192181.697cd6a363e514bff0d4ec954fc4034b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 697cd6a363e514bff0d4ec954fc4034b, NAME => 'TestAcidGuarantees,,1731751192181.697cd6a363e514bff0d4ec954fc4034b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T09:59:57,864 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=697cd6a363e514bff0d4ec954fc4034b, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T09:59:57,864 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36012 deadline: 1731751257862, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=697cd6a363e514bff0d4ec954fc4034b, server=037c58a19ddb,37267,1731751077844 2024-11-16T09:59:57,865 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731751192181.697cd6a363e514bff0d4ec954fc4034b., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 , the old value is region=TestAcidGuarantees,,1731751192181.697cd6a363e514bff0d4ec954fc4034b., hostname=037c58a19ddb,37267,1731751077844, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=697cd6a363e514bff0d4ec954fc4034b, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:59:57,865 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731751192181.697cd6a363e514bff0d4ec954fc4034b., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=697cd6a363e514bff0d4ec954fc4034b, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:59:57,865 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731751192181.697cd6a363e514bff0d4ec954fc4034b., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 because the exception is null or not the one we care about 2024-11-16T09:59:57,875 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=697cd6a363e514bff0d4ec954fc4034b, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T09:59:57,876 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36022 deadline: 1731751257873, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=697cd6a363e514bff0d4ec954fc4034b, server=037c58a19ddb,37267,1731751077844 2024-11-16T09:59:57,876 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=697cd6a363e514bff0d4ec954fc4034b, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T09:59:57,876 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731751192181.697cd6a363e514bff0d4ec954fc4034b., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 , the old value is region=TestAcidGuarantees,,1731751192181.697cd6a363e514bff0d4ec954fc4034b., hostname=037c58a19ddb,37267,1731751077844, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=697cd6a363e514bff0d4ec954fc4034b, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:59:57,876 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36058 deadline: 1731751257873, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=697cd6a363e514bff0d4ec954fc4034b, server=037c58a19ddb,37267,1731751077844 2024-11-16T09:59:57,876 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731751192181.697cd6a363e514bff0d4ec954fc4034b., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=697cd6a363e514bff0d4ec954fc4034b, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:59:57,876 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731751192181.697cd6a363e514bff0d4ec954fc4034b., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 because the exception is null or not the one we care about 2024-11-16T09:59:57,876 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731751192181.697cd6a363e514bff0d4ec954fc4034b., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 , the old value is region=TestAcidGuarantees,,1731751192181.697cd6a363e514bff0d4ec954fc4034b., hostname=037c58a19ddb,37267,1731751077844, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=697cd6a363e514bff0d4ec954fc4034b, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:59:57,876 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731751192181.697cd6a363e514bff0d4ec954fc4034b., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=697cd6a363e514bff0d4ec954fc4034b, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:59:57,876 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731751192181.697cd6a363e514bff0d4ec954fc4034b., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 because the exception is null or not the one we care about 2024-11-16T09:59:57,891 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=697cd6a363e514bff0d4ec954fc4034b, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T09:59:57,892 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36086 deadline: 1731751257883, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=697cd6a363e514bff0d4ec954fc4034b, server=037c58a19ddb,37267,1731751077844 2024-11-16T09:59:57,892 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731751192181.697cd6a363e514bff0d4ec954fc4034b., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 , the old value is region=TestAcidGuarantees,,1731751192181.697cd6a363e514bff0d4ec954fc4034b., hostname=037c58a19ddb,37267,1731751077844, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=697cd6a363e514bff0d4ec954fc4034b, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:59:57,892 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=697cd6a363e514bff0d4ec954fc4034b, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T09:59:57,892 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731751192181.697cd6a363e514bff0d4ec954fc4034b., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=697cd6a363e514bff0d4ec954fc4034b, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:59:57,892 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731751192181.697cd6a363e514bff0d4ec954fc4034b., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 because the exception is null or not the one we care about 2024-11-16T09:59:57,892 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36046 deadline: 1731751257883, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=697cd6a363e514bff0d4ec954fc4034b, server=037c58a19ddb,37267,1731751077844 2024-11-16T09:59:57,892 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731751192181.697cd6a363e514bff0d4ec954fc4034b., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 , the old value is region=TestAcidGuarantees,,1731751192181.697cd6a363e514bff0d4ec954fc4034b., hostname=037c58a19ddb,37267,1731751077844, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=697cd6a363e514bff0d4ec954fc4034b, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:59:57,892 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731751192181.697cd6a363e514bff0d4ec954fc4034b., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=697cd6a363e514bff0d4ec954fc4034b, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:59:57,892 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731751192181.697cd6a363e514bff0d4ec954fc4034b., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 because the exception is null or not the one we care about 2024-11-16T09:59:57,903 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=37267 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=122 2024-11-16T09:59:57,903 DEBUG [RS_FLUSH_OPERATIONS-regionserver/037c58a19ddb:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731751192181.697cd6a363e514bff0d4ec954fc4034b. 2024-11-16T09:59:57,903 DEBUG [RS_FLUSH_OPERATIONS-regionserver/037c58a19ddb:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731751192181.697cd6a363e514bff0d4ec954fc4034b. as already flushing 2024-11-16T09:59:57,904 DEBUG [RS_FLUSH_OPERATIONS-regionserver/037c58a19ddb:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731751192181.697cd6a363e514bff0d4ec954fc4034b. 2024-11-16T09:59:57,904 ERROR [RS_FLUSH_OPERATIONS-regionserver/037c58a19ddb:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] handler.RSProcedureHandler(58): pid=122 java.io.IOException: Unable to complete flush {ENCODED => 697cd6a363e514bff0d4ec954fc4034b, NAME => 'TestAcidGuarantees,,1731751192181.697cd6a363e514bff0d4ec954fc4034b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T09:59:57,904 DEBUG [RS_FLUSH_OPERATIONS-regionserver/037c58a19ddb:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=122 java.io.IOException: Unable to complete flush {ENCODED => 697cd6a363e514bff0d4ec954fc4034b, NAME => 'TestAcidGuarantees,,1731751192181.697cd6a363e514bff0d4ec954fc4034b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T09:59:57,904 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37877 {}] master.HMaster(4177): Remote procedure failed, pid=122 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 697cd6a363e514bff0d4ec954fc4034b, NAME => 'TestAcidGuarantees,,1731751192181.697cd6a363e514bff0d4ec954fc4034b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 697cd6a363e514bff0d4ec954fc4034b, NAME => 'TestAcidGuarantees,,1731751192181.697cd6a363e514bff0d4ec954fc4034b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T09:59:57,907 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=48 (bloomFilter=true), to=hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/697cd6a363e514bff0d4ec954fc4034b/.tmp/C/a7411494c98043698e28066a2f4b930b 2024-11-16T09:59:57,911 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/697cd6a363e514bff0d4ec954fc4034b/.tmp/A/38f40b75e4b343319edd57d2aca67982 as hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/697cd6a363e514bff0d4ec954fc4034b/A/38f40b75e4b343319edd57d2aca67982 2024-11-16T09:59:57,914 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/697cd6a363e514bff0d4ec954fc4034b/A/38f40b75e4b343319edd57d2aca67982, entries=200, sequenceid=48, filesize=14.0 K 2024-11-16T09:59:57,915 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/697cd6a363e514bff0d4ec954fc4034b/.tmp/B/19d2ff883d9a4216a9c2e142390c211d as hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/697cd6a363e514bff0d4ec954fc4034b/B/19d2ff883d9a4216a9c2e142390c211d 2024-11-16T09:59:57,918 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/697cd6a363e514bff0d4ec954fc4034b/B/19d2ff883d9a4216a9c2e142390c211d, entries=150, sequenceid=48, filesize=11.7 K 2024-11-16T09:59:57,919 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/697cd6a363e514bff0d4ec954fc4034b/.tmp/C/a7411494c98043698e28066a2f4b930b as hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/697cd6a363e514bff0d4ec954fc4034b/C/a7411494c98043698e28066a2f4b930b 2024-11-16T09:59:57,922 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/697cd6a363e514bff0d4ec954fc4034b/C/a7411494c98043698e28066a2f4b930b, entries=150, sequenceid=48, filesize=11.7 K 2024-11-16T09:59:57,923 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~141.33 KB/144720, currentSize=147.60 KB/151140 for 697cd6a363e514bff0d4ec954fc4034b in 1259ms, sequenceid=48, compaction requested=true 2024-11-16T09:59:57,923 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 697cd6a363e514bff0d4ec954fc4034b: 2024-11-16T09:59:57,923 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 697cd6a363e514bff0d4ec954fc4034b:A, priority=-2147483648, current under compaction store size is 1 2024-11-16T09:59:57,923 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-16T09:59:57,923 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 697cd6a363e514bff0d4ec954fc4034b:B, priority=-2147483648, current under compaction store size is 2 2024-11-16T09:59:57,923 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-16T09:59:57,923 DEBUG [RS:0;037c58a19ddb:37267-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-16T09:59:57,923 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 697cd6a363e514bff0d4ec954fc4034b:C, priority=-2147483648, current under compaction store size is 3 2024-11-16T09:59:57,923 DEBUG [RS:0;037c58a19ddb:37267-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-16T09:59:57,923 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-16T09:59:57,924 DEBUG [RS:0;037c58a19ddb:37267-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 38343 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-16T09:59:57,924 DEBUG [RS:0;037c58a19ddb:37267-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36003 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-16T09:59:57,924 DEBUG [RS:0;037c58a19ddb:37267-shortCompactions-0 {}] regionserver.HStore(1541): 697cd6a363e514bff0d4ec954fc4034b/A is initiating minor compaction (all files) 2024-11-16T09:59:57,924 DEBUG [RS:0;037c58a19ddb:37267-longCompactions-0 {}] regionserver.HStore(1541): 697cd6a363e514bff0d4ec954fc4034b/B is initiating minor compaction (all files) 2024-11-16T09:59:57,924 INFO [RS:0;037c58a19ddb:37267-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 697cd6a363e514bff0d4ec954fc4034b/A in TestAcidGuarantees,,1731751192181.697cd6a363e514bff0d4ec954fc4034b. 2024-11-16T09:59:57,924 INFO [RS:0;037c58a19ddb:37267-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 697cd6a363e514bff0d4ec954fc4034b/B in TestAcidGuarantees,,1731751192181.697cd6a363e514bff0d4ec954fc4034b. 2024-11-16T09:59:57,924 INFO [RS:0;037c58a19ddb:37267-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/697cd6a363e514bff0d4ec954fc4034b/A/656d8bd3c1f740f590bef993761e4820, hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/697cd6a363e514bff0d4ec954fc4034b/A/aedcfb03de7247b49de87f00d96a1a32, hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/697cd6a363e514bff0d4ec954fc4034b/A/38f40b75e4b343319edd57d2aca67982] into tmpdir=hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/697cd6a363e514bff0d4ec954fc4034b/.tmp, totalSize=37.4 K 2024-11-16T09:59:57,924 INFO [RS:0;037c58a19ddb:37267-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/697cd6a363e514bff0d4ec954fc4034b/B/b879c013fa994e78b1fe7c6327ecd121, hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/697cd6a363e514bff0d4ec954fc4034b/B/45b14ada2cc046159725de6299e3026e, hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/697cd6a363e514bff0d4ec954fc4034b/B/19d2ff883d9a4216a9c2e142390c211d] into tmpdir=hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/697cd6a363e514bff0d4ec954fc4034b/.tmp, totalSize=35.2 K 2024-11-16T09:59:57,925 DEBUG [RS:0;037c58a19ddb:37267-shortCompactions-0 {}] compactions.Compactor(225): Compacting 656d8bd3c1f740f590bef993761e4820, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=12, earliestPutTs=1731751194376 2024-11-16T09:59:57,925 DEBUG [RS:0;037c58a19ddb:37267-longCompactions-0 {}] compactions.Compactor(225): Compacting b879c013fa994e78b1fe7c6327ecd121, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=12, earliestPutTs=1731751194376 2024-11-16T09:59:57,925 DEBUG [RS:0;037c58a19ddb:37267-shortCompactions-0 {}] compactions.Compactor(225): Compacting aedcfb03de7247b49de87f00d96a1a32, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=37, earliestPutTs=1731751194469 2024-11-16T09:59:57,925 DEBUG [RS:0;037c58a19ddb:37267-longCompactions-0 {}] compactions.Compactor(225): Compacting 45b14ada2cc046159725de6299e3026e, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=37, earliestPutTs=1731751194469 2024-11-16T09:59:57,925 DEBUG [RS:0;037c58a19ddb:37267-longCompactions-0 {}] compactions.Compactor(225): Compacting 19d2ff883d9a4216a9c2e142390c211d, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=48, earliestPutTs=1731751196656 2024-11-16T09:59:57,926 DEBUG [RS:0;037c58a19ddb:37267-shortCompactions-0 {}] compactions.Compactor(225): Compacting 38f40b75e4b343319edd57d2aca67982, keycount=200, bloomtype=ROW, size=14.0 K, encoding=NONE, compression=NONE, seqNum=48, earliestPutTs=1731751196652 2024-11-16T09:59:57,937 INFO [RS:0;037c58a19ddb:37267-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 697cd6a363e514bff0d4ec954fc4034b#A#compaction#379 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-16T09:59:57,938 INFO [RS:0;037c58a19ddb:37267-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 697cd6a363e514bff0d4ec954fc4034b#B#compaction#380 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-16T09:59:57,938 DEBUG [RS:0;037c58a19ddb:37267-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/697cd6a363e514bff0d4ec954fc4034b/.tmp/A/824fa0f6583148b1a09539c16b299178 is 50, key is test_row_0/A:col10/1731751196662/Put/seqid=0 2024-11-16T09:59:57,938 DEBUG [RS:0;037c58a19ddb:37267-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/697cd6a363e514bff0d4ec954fc4034b/.tmp/B/20ca0a467e464862a5d2d9fef6e69a82 is 50, key is test_row_0/B:col10/1731751196662/Put/seqid=0 2024-11-16T09:59:57,945 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37591 is added to blk_1073742277_1453 (size=12104) 2024-11-16T09:59:57,946 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37591 is added to blk_1073742278_1454 (size=12104) 2024-11-16T09:59:57,953 DEBUG [RS:0;037c58a19ddb:37267-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/697cd6a363e514bff0d4ec954fc4034b/.tmp/A/824fa0f6583148b1a09539c16b299178 as hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/697cd6a363e514bff0d4ec954fc4034b/A/824fa0f6583148b1a09539c16b299178 2024-11-16T09:59:57,954 DEBUG [RS:0;037c58a19ddb:37267-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/697cd6a363e514bff0d4ec954fc4034b/.tmp/B/20ca0a467e464862a5d2d9fef6e69a82 as hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/697cd6a363e514bff0d4ec954fc4034b/B/20ca0a467e464862a5d2d9fef6e69a82 2024-11-16T09:59:57,959 INFO [RS:0;037c58a19ddb:37267-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 697cd6a363e514bff0d4ec954fc4034b/A of 697cd6a363e514bff0d4ec954fc4034b into 824fa0f6583148b1a09539c16b299178(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-16T09:59:57,959 DEBUG [RS:0;037c58a19ddb:37267-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 697cd6a363e514bff0d4ec954fc4034b: 2024-11-16T09:59:57,959 INFO [RS:0;037c58a19ddb:37267-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731751192181.697cd6a363e514bff0d4ec954fc4034b., storeName=697cd6a363e514bff0d4ec954fc4034b/A, priority=13, startTime=1731751197923; duration=0sec 2024-11-16T09:59:57,959 DEBUG [RS:0;037c58a19ddb:37267-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-16T09:59:57,959 DEBUG [RS:0;037c58a19ddb:37267-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 697cd6a363e514bff0d4ec954fc4034b:A 2024-11-16T09:59:57,959 DEBUG [RS:0;037c58a19ddb:37267-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-16T09:59:57,961 INFO [RS:0;037c58a19ddb:37267-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 697cd6a363e514bff0d4ec954fc4034b/B of 697cd6a363e514bff0d4ec954fc4034b into 20ca0a467e464862a5d2d9fef6e69a82(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-16T09:59:57,961 DEBUG [RS:0;037c58a19ddb:37267-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36003 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-16T09:59:57,961 DEBUG [RS:0;037c58a19ddb:37267-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 697cd6a363e514bff0d4ec954fc4034b: 2024-11-16T09:59:57,961 INFO [RS:0;037c58a19ddb:37267-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731751192181.697cd6a363e514bff0d4ec954fc4034b., storeName=697cd6a363e514bff0d4ec954fc4034b/B, priority=13, startTime=1731751197923; duration=0sec 2024-11-16T09:59:57,961 DEBUG [RS:0;037c58a19ddb:37267-shortCompactions-0 {}] regionserver.HStore(1541): 697cd6a363e514bff0d4ec954fc4034b/C is initiating minor compaction (all files) 2024-11-16T09:59:57,961 DEBUG [RS:0;037c58a19ddb:37267-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-16T09:59:57,961 DEBUG [RS:0;037c58a19ddb:37267-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 697cd6a363e514bff0d4ec954fc4034b:B 2024-11-16T09:59:57,961 INFO [RS:0;037c58a19ddb:37267-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 697cd6a363e514bff0d4ec954fc4034b/C in TestAcidGuarantees,,1731751192181.697cd6a363e514bff0d4ec954fc4034b. 2024-11-16T09:59:57,961 INFO [RS:0;037c58a19ddb:37267-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/697cd6a363e514bff0d4ec954fc4034b/C/a5314ebc7f464faba37fa36b02402253, hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/697cd6a363e514bff0d4ec954fc4034b/C/1180197cb1f34930b834b28b614575b2, hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/697cd6a363e514bff0d4ec954fc4034b/C/a7411494c98043698e28066a2f4b930b] into tmpdir=hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/697cd6a363e514bff0d4ec954fc4034b/.tmp, totalSize=35.2 K 2024-11-16T09:59:57,961 DEBUG [RS:0;037c58a19ddb:37267-shortCompactions-0 {}] compactions.Compactor(225): Compacting a5314ebc7f464faba37fa36b02402253, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=12, earliestPutTs=1731751194376 2024-11-16T09:59:57,962 DEBUG [RS:0;037c58a19ddb:37267-shortCompactions-0 {}] compactions.Compactor(225): Compacting 1180197cb1f34930b834b28b614575b2, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=37, earliestPutTs=1731751194469 2024-11-16T09:59:57,962 DEBUG [RS:0;037c58a19ddb:37267-shortCompactions-0 {}] compactions.Compactor(225): Compacting a7411494c98043698e28066a2f4b930b, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=48, earliestPutTs=1731751196656 2024-11-16T09:59:57,969 INFO [RS:0;037c58a19ddb:37267-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 697cd6a363e514bff0d4ec954fc4034b#C#compaction#381 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-16T09:59:57,970 DEBUG [RS:0;037c58a19ddb:37267-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/697cd6a363e514bff0d4ec954fc4034b/.tmp/C/2d47a6a059304795ba891d2763d74fad is 50, key is test_row_0/C:col10/1731751196662/Put/seqid=0 2024-11-16T09:59:57,973 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37591 is added to blk_1073742279_1455 (size=12104) 2024-11-16T09:59:57,979 DEBUG [RS:0;037c58a19ddb:37267-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/697cd6a363e514bff0d4ec954fc4034b/.tmp/C/2d47a6a059304795ba891d2763d74fad as hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/697cd6a363e514bff0d4ec954fc4034b/C/2d47a6a059304795ba891d2763d74fad 2024-11-16T09:59:57,984 INFO [RS:0;037c58a19ddb:37267-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 697cd6a363e514bff0d4ec954fc4034b/C of 697cd6a363e514bff0d4ec954fc4034b into 2d47a6a059304795ba891d2763d74fad(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-16T09:59:57,984 DEBUG [RS:0;037c58a19ddb:37267-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 697cd6a363e514bff0d4ec954fc4034b: 2024-11-16T09:59:57,984 INFO [RS:0;037c58a19ddb:37267-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731751192181.697cd6a363e514bff0d4ec954fc4034b., storeName=697cd6a363e514bff0d4ec954fc4034b/C, priority=13, startTime=1731751197923; duration=0sec 2024-11-16T09:59:57,984 DEBUG [RS:0;037c58a19ddb:37267-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-16T09:59:57,984 DEBUG [RS:0;037c58a19ddb:37267-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 697cd6a363e514bff0d4ec954fc4034b:C 2024-11-16T09:59:58,056 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=37267 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=122 2024-11-16T09:59:58,056 DEBUG [RS_FLUSH_OPERATIONS-regionserver/037c58a19ddb:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731751192181.697cd6a363e514bff0d4ec954fc4034b. 2024-11-16T09:59:58,057 INFO [RS_FLUSH_OPERATIONS-regionserver/037c58a19ddb:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.HRegion(2902): Flushing 697cd6a363e514bff0d4ec954fc4034b 3/3 column families, dataSize=147.60 KB heapSize=387.47 KB 2024-11-16T09:59:58,061 DEBUG [RS_FLUSH_OPERATIONS-regionserver/037c58a19ddb:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/697cd6a363e514bff0d4ec954fc4034b/.tmp/A/fbfadead6dcb499097ec150548f8c3a1 is 50, key is test_row_0/A:col10/1731751196708/Put/seqid=0 2024-11-16T09:59:58,065 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37591 is added to blk_1073742280_1456 (size=12001) 2024-11-16T09:59:58,465 INFO [RS_FLUSH_OPERATIONS-regionserver/037c58a19ddb:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=76 (bloomFilter=true), to=hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/697cd6a363e514bff0d4ec954fc4034b/.tmp/A/fbfadead6dcb499097ec150548f8c3a1 2024-11-16T09:59:58,472 DEBUG [RS_FLUSH_OPERATIONS-regionserver/037c58a19ddb:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/697cd6a363e514bff0d4ec954fc4034b/.tmp/B/e9af6ad3179a4ff38bce9dafdfed880b is 50, key is test_row_0/B:col10/1731751196708/Put/seqid=0 2024-11-16T09:59:58,482 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37591 is added to blk_1073742281_1457 (size=12001) 2024-11-16T09:59:58,483 INFO [RS_FLUSH_OPERATIONS-regionserver/037c58a19ddb:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=76 (bloomFilter=true), to=hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/697cd6a363e514bff0d4ec954fc4034b/.tmp/B/e9af6ad3179a4ff38bce9dafdfed880b 2024-11-16T09:59:58,490 DEBUG [RS_FLUSH_OPERATIONS-regionserver/037c58a19ddb:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/697cd6a363e514bff0d4ec954fc4034b/.tmp/C/6776f39b530c4e44bc12e7f8fc2004fc is 50, key is test_row_0/C:col10/1731751196708/Put/seqid=0 2024-11-16T09:59:58,495 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37591 is added to blk_1073742282_1458 (size=12001) 2024-11-16T09:59:58,673 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37877 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=121 2024-11-16T09:59:58,876 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] regionserver.HRegion(8855): Flush requested on 697cd6a363e514bff0d4ec954fc4034b 2024-11-16T09:59:58,876 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731751192181.697cd6a363e514bff0d4ec954fc4034b. as already flushing 2024-11-16T09:59:58,895 INFO [RS_FLUSH_OPERATIONS-regionserver/037c58a19ddb:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=76 (bloomFilter=true), to=hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/697cd6a363e514bff0d4ec954fc4034b/.tmp/C/6776f39b530c4e44bc12e7f8fc2004fc 2024-11-16T09:59:58,900 DEBUG [RS_FLUSH_OPERATIONS-regionserver/037c58a19ddb:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/697cd6a363e514bff0d4ec954fc4034b/.tmp/A/fbfadead6dcb499097ec150548f8c3a1 as hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/697cd6a363e514bff0d4ec954fc4034b/A/fbfadead6dcb499097ec150548f8c3a1 2024-11-16T09:59:58,903 INFO [RS_FLUSH_OPERATIONS-regionserver/037c58a19ddb:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/697cd6a363e514bff0d4ec954fc4034b/A/fbfadead6dcb499097ec150548f8c3a1, entries=150, sequenceid=76, filesize=11.7 K 2024-11-16T09:59:58,904 DEBUG [RS_FLUSH_OPERATIONS-regionserver/037c58a19ddb:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/697cd6a363e514bff0d4ec954fc4034b/.tmp/B/e9af6ad3179a4ff38bce9dafdfed880b as hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/697cd6a363e514bff0d4ec954fc4034b/B/e9af6ad3179a4ff38bce9dafdfed880b 2024-11-16T09:59:58,904 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=697cd6a363e514bff0d4ec954fc4034b, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T09:59:58,904 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36012 deadline: 1731751258896, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=697cd6a363e514bff0d4ec954fc4034b, server=037c58a19ddb,37267,1731751077844 2024-11-16T09:59:58,904 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731751192181.697cd6a363e514bff0d4ec954fc4034b., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 , the old value is region=TestAcidGuarantees,,1731751192181.697cd6a363e514bff0d4ec954fc4034b., hostname=037c58a19ddb,37267,1731751077844, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=697cd6a363e514bff0d4ec954fc4034b, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:59:58,905 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731751192181.697cd6a363e514bff0d4ec954fc4034b., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=697cd6a363e514bff0d4ec954fc4034b, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:59:58,905 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731751192181.697cd6a363e514bff0d4ec954fc4034b., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 because the exception is null or not the one we care about 2024-11-16T09:59:58,907 INFO [RS_FLUSH_OPERATIONS-regionserver/037c58a19ddb:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/697cd6a363e514bff0d4ec954fc4034b/B/e9af6ad3179a4ff38bce9dafdfed880b, entries=150, sequenceid=76, filesize=11.7 K 2024-11-16T09:59:58,907 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=697cd6a363e514bff0d4ec954fc4034b, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T09:59:58,908 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36086 deadline: 1731751258903, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=697cd6a363e514bff0d4ec954fc4034b, server=037c58a19ddb,37267,1731751077844 2024-11-16T09:59:58,908 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=697cd6a363e514bff0d4ec954fc4034b, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T09:59:58,908 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36022 deadline: 1731751258903, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=697cd6a363e514bff0d4ec954fc4034b, server=037c58a19ddb,37267,1731751077844 2024-11-16T09:59:58,908 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=697cd6a363e514bff0d4ec954fc4034b, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T09:59:58,908 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36046 deadline: 1731751258903, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=697cd6a363e514bff0d4ec954fc4034b, server=037c58a19ddb,37267,1731751077844 2024-11-16T09:59:58,908 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731751192181.697cd6a363e514bff0d4ec954fc4034b., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 , the old value is region=TestAcidGuarantees,,1731751192181.697cd6a363e514bff0d4ec954fc4034b., hostname=037c58a19ddb,37267,1731751077844, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=697cd6a363e514bff0d4ec954fc4034b, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:59:58,908 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731751192181.697cd6a363e514bff0d4ec954fc4034b., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=697cd6a363e514bff0d4ec954fc4034b, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:59:58,908 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731751192181.697cd6a363e514bff0d4ec954fc4034b., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 because the exception is null or not the one we care about 2024-11-16T09:59:58,908 WARN [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRpcRetryingCaller(177): Call to 037c58a19ddb,37267,1731751077844 for 'test_row_2' in 697cd6a363e514bff0d4ec954fc4034b of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2206 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=697cd6a363e514bff0d4ec954fc4034b, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor44.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=697cd6a363e514bff0d4ec954fc4034b, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-16T09:59:58,908 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731751192181.697cd6a363e514bff0d4ec954fc4034b., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 , the old value is region=TestAcidGuarantees,,1731751192181.697cd6a363e514bff0d4ec954fc4034b., hostname=037c58a19ddb,37267,1731751077844, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=697cd6a363e514bff0d4ec954fc4034b, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:59:58,908 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731751192181.697cd6a363e514bff0d4ec954fc4034b., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=697cd6a363e514bff0d4ec954fc4034b, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:59:58,908 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731751192181.697cd6a363e514bff0d4ec954fc4034b., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 because the exception is null or not the one we care about 2024-11-16T09:59:58,908 WARN [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRpcRetryingCaller(177): Call to 037c58a19ddb,37267,1731751077844 for 'test_row_2' in 697cd6a363e514bff0d4ec954fc4034b of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2195 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=697cd6a363e514bff0d4ec954fc4034b, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor44.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=697cd6a363e514bff0d4ec954fc4034b, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-16T09:59:58,909 DEBUG [RS_FLUSH_OPERATIONS-regionserver/037c58a19ddb:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/697cd6a363e514bff0d4ec954fc4034b/.tmp/C/6776f39b530c4e44bc12e7f8fc2004fc as hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/697cd6a363e514bff0d4ec954fc4034b/C/6776f39b530c4e44bc12e7f8fc2004fc 2024-11-16T09:59:58,909 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731751192181.697cd6a363e514bff0d4ec954fc4034b., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 , the old value is region=TestAcidGuarantees,,1731751192181.697cd6a363e514bff0d4ec954fc4034b., hostname=037c58a19ddb,37267,1731751077844, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=697cd6a363e514bff0d4ec954fc4034b, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:59:58,909 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731751192181.697cd6a363e514bff0d4ec954fc4034b., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=697cd6a363e514bff0d4ec954fc4034b, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:59:58,909 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731751192181.697cd6a363e514bff0d4ec954fc4034b., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 because the exception is null or not the one we care about 2024-11-16T09:59:58,912 INFO [RS_FLUSH_OPERATIONS-regionserver/037c58a19ddb:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/697cd6a363e514bff0d4ec954fc4034b/C/6776f39b530c4e44bc12e7f8fc2004fc, entries=150, sequenceid=76, filesize=11.7 K 2024-11-16T09:59:58,913 INFO [RS_FLUSH_OPERATIONS-regionserver/037c58a19ddb:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.HRegion(3140): Finished flush of dataSize ~147.60 KB/151140, heapSize ~387.42 KB/396720, currentSize=53.67 KB/54960 for 697cd6a363e514bff0d4ec954fc4034b in 857ms, sequenceid=76, compaction requested=false 2024-11-16T09:59:58,913 DEBUG [RS_FLUSH_OPERATIONS-regionserver/037c58a19ddb:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.HRegion(2603): Flush status journal for 697cd6a363e514bff0d4ec954fc4034b: 2024-11-16T09:59:58,913 DEBUG [RS_FLUSH_OPERATIONS-regionserver/037c58a19ddb:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731751192181.697cd6a363e514bff0d4ec954fc4034b. 2024-11-16T09:59:58,913 DEBUG [RS_FLUSH_OPERATIONS-regionserver/037c58a19ddb:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=122 2024-11-16T09:59:58,913 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37877 {}] master.HMaster(4169): Remote procedure done, pid=122 2024-11-16T09:59:58,914 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] regionserver.HRegion(8855): Flush requested on 697cd6a363e514bff0d4ec954fc4034b 2024-11-16T09:59:58,914 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-16T09:59:58,915 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 697cd6a363e514bff0d4ec954fc4034b 3/3 column families, dataSize=60.38 KB heapSize=158.95 KB 2024-11-16T09:59:58,917 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=122, resume processing ppid=121 2024-11-16T09:59:58,917 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=122, ppid=121, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.3870 sec 2024-11-16T09:59:58,918 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/697cd6a363e514bff0d4ec954fc4034b/.tmp/A/693c6bb535ea4f67b3213cf70d77313d is 50, key is test_row_0/A:col10/1731751198913/Put/seqid=0 2024-11-16T09:59:58,918 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=121, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=121, table=TestAcidGuarantees in 2.3930 sec 2024-11-16T09:59:58,921 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37591 is added to blk_1073742283_1459 (size=14341) 2024-11-16T09:59:58,922 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=88 (bloomFilter=true), to=hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/697cd6a363e514bff0d4ec954fc4034b/.tmp/A/693c6bb535ea4f67b3213cf70d77313d 2024-11-16T09:59:58,928 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/697cd6a363e514bff0d4ec954fc4034b/.tmp/B/0096a10bcf1b406dba763c0e979e5c5e is 50, key is test_row_0/B:col10/1731751198913/Put/seqid=0 2024-11-16T09:59:58,932 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37591 is added to blk_1073742284_1460 (size=12001) 2024-11-16T09:59:58,933 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=88 (bloomFilter=true), to=hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/697cd6a363e514bff0d4ec954fc4034b/.tmp/B/0096a10bcf1b406dba763c0e979e5c5e 2024-11-16T09:59:58,939 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/697cd6a363e514bff0d4ec954fc4034b/.tmp/C/37cbfa8ef4e44d8da5b85d7a0ce5089d is 50, key is test_row_0/C:col10/1731751198913/Put/seqid=0 2024-11-16T09:59:58,942 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37591 is added to blk_1073742285_1461 (size=12001) 2024-11-16T09:59:59,039 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=697cd6a363e514bff0d4ec954fc4034b, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T09:59:59,039 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36012 deadline: 1731751259036, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=697cd6a363e514bff0d4ec954fc4034b, server=037c58a19ddb,37267,1731751077844 2024-11-16T09:59:59,039 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=697cd6a363e514bff0d4ec954fc4034b, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T09:59:59,039 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731751192181.697cd6a363e514bff0d4ec954fc4034b., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 , the old value is region=TestAcidGuarantees,,1731751192181.697cd6a363e514bff0d4ec954fc4034b., hostname=037c58a19ddb,37267,1731751077844, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=697cd6a363e514bff0d4ec954fc4034b, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:59:59,039 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731751192181.697cd6a363e514bff0d4ec954fc4034b., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=697cd6a363e514bff0d4ec954fc4034b, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:59:59,039 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36022 deadline: 1731751259036, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=697cd6a363e514bff0d4ec954fc4034b, server=037c58a19ddb,37267,1731751077844 2024-11-16T09:59:59,039 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731751192181.697cd6a363e514bff0d4ec954fc4034b., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 because the exception is null or not the one we care about 2024-11-16T09:59:59,039 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731751192181.697cd6a363e514bff0d4ec954fc4034b., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 , the old value is region=TestAcidGuarantees,,1731751192181.697cd6a363e514bff0d4ec954fc4034b., hostname=037c58a19ddb,37267,1731751077844, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=697cd6a363e514bff0d4ec954fc4034b, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:59:59,039 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731751192181.697cd6a363e514bff0d4ec954fc4034b., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=697cd6a363e514bff0d4ec954fc4034b, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:59:59,039 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731751192181.697cd6a363e514bff0d4ec954fc4034b., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 because the exception is null or not the one we care about 2024-11-16T09:59:59,047 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=697cd6a363e514bff0d4ec954fc4034b, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T09:59:59,047 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36058 deadline: 1731751259037, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=697cd6a363e514bff0d4ec954fc4034b, server=037c58a19ddb,37267,1731751077844 2024-11-16T09:59:59,048 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731751192181.697cd6a363e514bff0d4ec954fc4034b., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 , the old value is region=TestAcidGuarantees,,1731751192181.697cd6a363e514bff0d4ec954fc4034b., hostname=037c58a19ddb,37267,1731751077844, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=697cd6a363e514bff0d4ec954fc4034b, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:59:59,048 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731751192181.697cd6a363e514bff0d4ec954fc4034b., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=697cd6a363e514bff0d4ec954fc4034b, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:59:59,048 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731751192181.697cd6a363e514bff0d4ec954fc4034b., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 because the exception is null or not the one we care about 2024-11-16T09:59:59,149 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=697cd6a363e514bff0d4ec954fc4034b, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T09:59:59,149 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36012 deadline: 1731751259143, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=697cd6a363e514bff0d4ec954fc4034b, server=037c58a19ddb,37267,1731751077844 2024-11-16T09:59:59,149 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=697cd6a363e514bff0d4ec954fc4034b, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T09:59:59,149 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731751192181.697cd6a363e514bff0d4ec954fc4034b., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 , the old value is region=TestAcidGuarantees,,1731751192181.697cd6a363e514bff0d4ec954fc4034b., hostname=037c58a19ddb,37267,1731751077844, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=697cd6a363e514bff0d4ec954fc4034b, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:59:59,149 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731751192181.697cd6a363e514bff0d4ec954fc4034b., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=697cd6a363e514bff0d4ec954fc4034b, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:59:59,150 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731751192181.697cd6a363e514bff0d4ec954fc4034b., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 because the exception is null or not the one we care about 2024-11-16T09:59:59,150 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36022 deadline: 1731751259143, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=697cd6a363e514bff0d4ec954fc4034b, server=037c58a19ddb,37267,1731751077844 2024-11-16T09:59:59,150 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731751192181.697cd6a363e514bff0d4ec954fc4034b., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 , the old value is region=TestAcidGuarantees,,1731751192181.697cd6a363e514bff0d4ec954fc4034b., hostname=037c58a19ddb,37267,1731751077844, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=697cd6a363e514bff0d4ec954fc4034b, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:59:59,150 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731751192181.697cd6a363e514bff0d4ec954fc4034b., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=697cd6a363e514bff0d4ec954fc4034b, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:59:59,150 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731751192181.697cd6a363e514bff0d4ec954fc4034b., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 because the exception is null or not the one we care about 2024-11-16T09:59:59,160 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=697cd6a363e514bff0d4ec954fc4034b, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T09:59:59,160 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36058 deadline: 1731751259152, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=697cd6a363e514bff0d4ec954fc4034b, server=037c58a19ddb,37267,1731751077844 2024-11-16T09:59:59,160 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731751192181.697cd6a363e514bff0d4ec954fc4034b., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 , the old value is region=TestAcidGuarantees,,1731751192181.697cd6a363e514bff0d4ec954fc4034b., hostname=037c58a19ddb,37267,1731751077844, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=697cd6a363e514bff0d4ec954fc4034b, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:59:59,160 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731751192181.697cd6a363e514bff0d4ec954fc4034b., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=697cd6a363e514bff0d4ec954fc4034b, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:59:59,160 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731751192181.697cd6a363e514bff0d4ec954fc4034b., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 because the exception is null or not the one we care about 2024-11-16T09:59:59,342 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=88 (bloomFilter=true), to=hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/697cd6a363e514bff0d4ec954fc4034b/.tmp/C/37cbfa8ef4e44d8da5b85d7a0ce5089d 2024-11-16T09:59:59,346 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/697cd6a363e514bff0d4ec954fc4034b/.tmp/A/693c6bb535ea4f67b3213cf70d77313d as hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/697cd6a363e514bff0d4ec954fc4034b/A/693c6bb535ea4f67b3213cf70d77313d 2024-11-16T09:59:59,350 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/697cd6a363e514bff0d4ec954fc4034b/A/693c6bb535ea4f67b3213cf70d77313d, entries=200, sequenceid=88, filesize=14.0 K 2024-11-16T09:59:59,351 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/697cd6a363e514bff0d4ec954fc4034b/.tmp/B/0096a10bcf1b406dba763c0e979e5c5e as hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/697cd6a363e514bff0d4ec954fc4034b/B/0096a10bcf1b406dba763c0e979e5c5e 2024-11-16T09:59:59,354 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/697cd6a363e514bff0d4ec954fc4034b/B/0096a10bcf1b406dba763c0e979e5c5e, entries=150, sequenceid=88, filesize=11.7 K 2024-11-16T09:59:59,354 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/697cd6a363e514bff0d4ec954fc4034b/.tmp/C/37cbfa8ef4e44d8da5b85d7a0ce5089d as hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/697cd6a363e514bff0d4ec954fc4034b/C/37cbfa8ef4e44d8da5b85d7a0ce5089d 2024-11-16T09:59:59,357 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/697cd6a363e514bff0d4ec954fc4034b/C/37cbfa8ef4e44d8da5b85d7a0ce5089d, entries=150, sequenceid=88, filesize=11.7 K 2024-11-16T09:59:59,358 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~158.91 KB/162720, currentSize=140.89 KB/144270 for 697cd6a363e514bff0d4ec954fc4034b in 443ms, sequenceid=88, compaction requested=true 2024-11-16T09:59:59,358 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 697cd6a363e514bff0d4ec954fc4034b: 2024-11-16T09:59:59,358 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 697cd6a363e514bff0d4ec954fc4034b:A, priority=-2147483648, current under compaction store size is 1 2024-11-16T09:59:59,358 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-16T09:59:59,358 DEBUG [RS:0;037c58a19ddb:37267-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-16T09:59:59,358 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 697cd6a363e514bff0d4ec954fc4034b:B, priority=-2147483648, current under compaction store size is 2 2024-11-16T09:59:59,358 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-16T09:59:59,358 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 697cd6a363e514bff0d4ec954fc4034b:C, priority=-2147483648, current under compaction store size is 3 2024-11-16T09:59:59,358 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-16T09:59:59,358 DEBUG [RS:0;037c58a19ddb:37267-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-16T09:59:59,359 DEBUG [RS:0;037c58a19ddb:37267-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 38446 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-16T09:59:59,359 DEBUG [RS:0;037c58a19ddb:37267-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36106 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-16T09:59:59,359 DEBUG [RS:0;037c58a19ddb:37267-shortCompactions-0 {}] regionserver.HStore(1541): 697cd6a363e514bff0d4ec954fc4034b/A is initiating minor compaction (all files) 2024-11-16T09:59:59,359 DEBUG [RS:0;037c58a19ddb:37267-longCompactions-0 {}] regionserver.HStore(1541): 697cd6a363e514bff0d4ec954fc4034b/B is initiating minor compaction (all files) 2024-11-16T09:59:59,359 INFO [RS:0;037c58a19ddb:37267-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 697cd6a363e514bff0d4ec954fc4034b/A in TestAcidGuarantees,,1731751192181.697cd6a363e514bff0d4ec954fc4034b. 2024-11-16T09:59:59,359 INFO [RS:0;037c58a19ddb:37267-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 697cd6a363e514bff0d4ec954fc4034b/B in TestAcidGuarantees,,1731751192181.697cd6a363e514bff0d4ec954fc4034b. 2024-11-16T09:59:59,359 INFO [RS:0;037c58a19ddb:37267-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/697cd6a363e514bff0d4ec954fc4034b/A/824fa0f6583148b1a09539c16b299178, hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/697cd6a363e514bff0d4ec954fc4034b/A/fbfadead6dcb499097ec150548f8c3a1, hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/697cd6a363e514bff0d4ec954fc4034b/A/693c6bb535ea4f67b3213cf70d77313d] into tmpdir=hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/697cd6a363e514bff0d4ec954fc4034b/.tmp, totalSize=37.5 K 2024-11-16T09:59:59,359 INFO [RS:0;037c58a19ddb:37267-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/697cd6a363e514bff0d4ec954fc4034b/B/20ca0a467e464862a5d2d9fef6e69a82, hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/697cd6a363e514bff0d4ec954fc4034b/B/e9af6ad3179a4ff38bce9dafdfed880b, hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/697cd6a363e514bff0d4ec954fc4034b/B/0096a10bcf1b406dba763c0e979e5c5e] into tmpdir=hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/697cd6a363e514bff0d4ec954fc4034b/.tmp, totalSize=35.3 K 2024-11-16T09:59:59,359 DEBUG [RS:0;037c58a19ddb:37267-shortCompactions-0 {}] compactions.Compactor(225): Compacting 824fa0f6583148b1a09539c16b299178, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=48, earliestPutTs=1731751196656 2024-11-16T09:59:59,359 DEBUG [RS:0;037c58a19ddb:37267-longCompactions-0 {}] compactions.Compactor(225): Compacting 20ca0a467e464862a5d2d9fef6e69a82, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=48, earliestPutTs=1731751196656 2024-11-16T09:59:59,360 DEBUG [RS:0;037c58a19ddb:37267-shortCompactions-0 {}] compactions.Compactor(225): Compacting fbfadead6dcb499097ec150548f8c3a1, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=76, earliestPutTs=1731751196708 2024-11-16T09:59:59,360 DEBUG [RS:0;037c58a19ddb:37267-longCompactions-0 {}] compactions.Compactor(225): Compacting e9af6ad3179a4ff38bce9dafdfed880b, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=76, earliestPutTs=1731751196708 2024-11-16T09:59:59,360 DEBUG [RS:0;037c58a19ddb:37267-longCompactions-0 {}] compactions.Compactor(225): Compacting 0096a10bcf1b406dba763c0e979e5c5e, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=88, earliestPutTs=1731751198888 2024-11-16T09:59:59,360 DEBUG [RS:0;037c58a19ddb:37267-shortCompactions-0 {}] compactions.Compactor(225): Compacting 693c6bb535ea4f67b3213cf70d77313d, keycount=200, bloomtype=ROW, size=14.0 K, encoding=NONE, compression=NONE, seqNum=88, earliestPutTs=1731751198888 2024-11-16T09:59:59,361 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] regionserver.HRegion(8855): Flush requested on 697cd6a363e514bff0d4ec954fc4034b 2024-11-16T09:59:59,361 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-16T09:59:59,362 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 697cd6a363e514bff0d4ec954fc4034b 3/3 column families, dataSize=147.60 KB heapSize=387.47 KB 2024-11-16T09:59:59,368 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/697cd6a363e514bff0d4ec954fc4034b/.tmp/A/93f09bcb15ca453ba30359031b06d38d is 50, key is test_row_0/A:col10/1731751199022/Put/seqid=0 2024-11-16T09:59:59,368 INFO [RS:0;037c58a19ddb:37267-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 697cd6a363e514bff0d4ec954fc4034b#B#compaction#389 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-16T09:59:59,369 DEBUG [RS:0;037c58a19ddb:37267-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/697cd6a363e514bff0d4ec954fc4034b/.tmp/B/9f94221481804f42b846e9e9614916f6 is 50, key is test_row_0/B:col10/1731751198913/Put/seqid=0 2024-11-16T09:59:59,369 INFO [RS:0;037c58a19ddb:37267-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 697cd6a363e514bff0d4ec954fc4034b#A#compaction#390 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-16T09:59:59,370 DEBUG [RS:0;037c58a19ddb:37267-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/697cd6a363e514bff0d4ec954fc4034b/.tmp/A/b0882ffe2ec9422087f061171d91e3bc is 50, key is test_row_0/A:col10/1731751198913/Put/seqid=0 2024-11-16T09:59:59,373 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37591 is added to blk_1073742286_1462 (size=14341) 2024-11-16T09:59:59,384 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37591 is added to blk_1073742287_1463 (size=12207) 2024-11-16T09:59:59,389 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37591 is added to blk_1073742288_1464 (size=12207) 2024-11-16T09:59:59,399 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=697cd6a363e514bff0d4ec954fc4034b, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T09:59:59,399 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36012 deadline: 1731751259387, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=697cd6a363e514bff0d4ec954fc4034b, server=037c58a19ddb,37267,1731751077844 2024-11-16T09:59:59,399 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731751192181.697cd6a363e514bff0d4ec954fc4034b., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 , the old value is region=TestAcidGuarantees,,1731751192181.697cd6a363e514bff0d4ec954fc4034b., hostname=037c58a19ddb,37267,1731751077844, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=697cd6a363e514bff0d4ec954fc4034b, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:59:59,399 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=697cd6a363e514bff0d4ec954fc4034b, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T09:59:59,399 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731751192181.697cd6a363e514bff0d4ec954fc4034b., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=697cd6a363e514bff0d4ec954fc4034b, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:59:59,399 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731751192181.697cd6a363e514bff0d4ec954fc4034b., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 because the exception is null or not the one we care about 2024-11-16T09:59:59,399 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36022 deadline: 1731751259388, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=697cd6a363e514bff0d4ec954fc4034b, server=037c58a19ddb,37267,1731751077844 2024-11-16T09:59:59,400 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731751192181.697cd6a363e514bff0d4ec954fc4034b., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 , the old value is region=TestAcidGuarantees,,1731751192181.697cd6a363e514bff0d4ec954fc4034b., hostname=037c58a19ddb,37267,1731751077844, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=697cd6a363e514bff0d4ec954fc4034b, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:59:59,400 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731751192181.697cd6a363e514bff0d4ec954fc4034b., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=697cd6a363e514bff0d4ec954fc4034b, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:59:59,400 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731751192181.697cd6a363e514bff0d4ec954fc4034b., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 because the exception is null or not the one we care about 2024-11-16T09:59:59,403 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=697cd6a363e514bff0d4ec954fc4034b, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T09:59:59,403 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36058 deadline: 1731751259399, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=697cd6a363e514bff0d4ec954fc4034b, server=037c58a19ddb,37267,1731751077844 2024-11-16T09:59:59,403 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731751192181.697cd6a363e514bff0d4ec954fc4034b., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 , the old value is region=TestAcidGuarantees,,1731751192181.697cd6a363e514bff0d4ec954fc4034b., hostname=037c58a19ddb,37267,1731751077844, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=697cd6a363e514bff0d4ec954fc4034b, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:59:59,403 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731751192181.697cd6a363e514bff0d4ec954fc4034b., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=697cd6a363e514bff0d4ec954fc4034b, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:59:59,404 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731751192181.697cd6a363e514bff0d4ec954fc4034b., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 because the exception is null or not the one we care about 2024-11-16T09:59:59,509 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=697cd6a363e514bff0d4ec954fc4034b, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T09:59:59,509 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36012 deadline: 1731751259502, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=697cd6a363e514bff0d4ec954fc4034b, server=037c58a19ddb,37267,1731751077844 2024-11-16T09:59:59,509 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=697cd6a363e514bff0d4ec954fc4034b, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T09:59:59,509 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731751192181.697cd6a363e514bff0d4ec954fc4034b., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 , the old value is region=TestAcidGuarantees,,1731751192181.697cd6a363e514bff0d4ec954fc4034b., hostname=037c58a19ddb,37267,1731751077844, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=697cd6a363e514bff0d4ec954fc4034b, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:59:59,509 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731751192181.697cd6a363e514bff0d4ec954fc4034b., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=697cd6a363e514bff0d4ec954fc4034b, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:59:59,509 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36022 deadline: 1731751259503, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=697cd6a363e514bff0d4ec954fc4034b, server=037c58a19ddb,37267,1731751077844 2024-11-16T09:59:59,509 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731751192181.697cd6a363e514bff0d4ec954fc4034b., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 because the exception is null or not the one we care about 2024-11-16T09:59:59,509 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731751192181.697cd6a363e514bff0d4ec954fc4034b., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 , the old value is region=TestAcidGuarantees,,1731751192181.697cd6a363e514bff0d4ec954fc4034b., hostname=037c58a19ddb,37267,1731751077844, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=697cd6a363e514bff0d4ec954fc4034b, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:59:59,509 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731751192181.697cd6a363e514bff0d4ec954fc4034b., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=697cd6a363e514bff0d4ec954fc4034b, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:59:59,509 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731751192181.697cd6a363e514bff0d4ec954fc4034b., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 because the exception is null or not the one we care about 2024-11-16T09:59:59,520 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=697cd6a363e514bff0d4ec954fc4034b, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T09:59:59,520 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36058 deadline: 1731751259513, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=697cd6a363e514bff0d4ec954fc4034b, server=037c58a19ddb,37267,1731751077844 2024-11-16T09:59:59,521 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731751192181.697cd6a363e514bff0d4ec954fc4034b., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 , the old value is region=TestAcidGuarantees,,1731751192181.697cd6a363e514bff0d4ec954fc4034b., hostname=037c58a19ddb,37267,1731751077844, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=697cd6a363e514bff0d4ec954fc4034b, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:59:59,521 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731751192181.697cd6a363e514bff0d4ec954fc4034b., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=697cd6a363e514bff0d4ec954fc4034b, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:59:59,521 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731751192181.697cd6a363e514bff0d4ec954fc4034b., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 because the exception is null or not the one we care about 2024-11-16T09:59:59,717 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=697cd6a363e514bff0d4ec954fc4034b, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T09:59:59,717 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36012 deadline: 1731751259712, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=697cd6a363e514bff0d4ec954fc4034b, server=037c58a19ddb,37267,1731751077844 2024-11-16T09:59:59,717 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=697cd6a363e514bff0d4ec954fc4034b, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T09:59:59,717 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731751192181.697cd6a363e514bff0d4ec954fc4034b., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 , the old value is region=TestAcidGuarantees,,1731751192181.697cd6a363e514bff0d4ec954fc4034b., hostname=037c58a19ddb,37267,1731751077844, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=697cd6a363e514bff0d4ec954fc4034b, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:59:59,717 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36022 deadline: 1731751259712, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=697cd6a363e514bff0d4ec954fc4034b, server=037c58a19ddb,37267,1731751077844 2024-11-16T09:59:59,717 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731751192181.697cd6a363e514bff0d4ec954fc4034b., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=697cd6a363e514bff0d4ec954fc4034b, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:59:59,717 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731751192181.697cd6a363e514bff0d4ec954fc4034b., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 because the exception is null or not the one we care about 2024-11-16T09:59:59,717 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731751192181.697cd6a363e514bff0d4ec954fc4034b., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 , the old value is region=TestAcidGuarantees,,1731751192181.697cd6a363e514bff0d4ec954fc4034b., hostname=037c58a19ddb,37267,1731751077844, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=697cd6a363e514bff0d4ec954fc4034b, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:59:59,717 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731751192181.697cd6a363e514bff0d4ec954fc4034b., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=697cd6a363e514bff0d4ec954fc4034b, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:59:59,717 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731751192181.697cd6a363e514bff0d4ec954fc4034b., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 because the exception is null or not the one we care about 2024-11-16T09:59:59,735 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=697cd6a363e514bff0d4ec954fc4034b, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T09:59:59,735 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36058 deadline: 1731751259733, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=697cd6a363e514bff0d4ec954fc4034b, server=037c58a19ddb,37267,1731751077844 2024-11-16T09:59:59,736 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731751192181.697cd6a363e514bff0d4ec954fc4034b., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 , the old value is region=TestAcidGuarantees,,1731751192181.697cd6a363e514bff0d4ec954fc4034b., hostname=037c58a19ddb,37267,1731751077844, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=697cd6a363e514bff0d4ec954fc4034b, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:59:59,736 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731751192181.697cd6a363e514bff0d4ec954fc4034b., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=697cd6a363e514bff0d4ec954fc4034b, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T09:59:59,736 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731751192181.697cd6a363e514bff0d4ec954fc4034b., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 because the exception is null or not the one we care about 2024-11-16T09:59:59,774 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=113 (bloomFilter=true), to=hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/697cd6a363e514bff0d4ec954fc4034b/.tmp/A/93f09bcb15ca453ba30359031b06d38d 2024-11-16T09:59:59,783 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/697cd6a363e514bff0d4ec954fc4034b/.tmp/B/34019d6dca5944bb901ff94f64922d8a is 50, key is test_row_0/B:col10/1731751199022/Put/seqid=0 2024-11-16T09:59:59,787 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37591 is added to blk_1073742289_1465 (size=12001) 2024-11-16T09:59:59,789 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=113 (bloomFilter=true), to=hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/697cd6a363e514bff0d4ec954fc4034b/.tmp/B/34019d6dca5944bb901ff94f64922d8a 2024-11-16T09:59:59,789 DEBUG [RS:0;037c58a19ddb:37267-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/697cd6a363e514bff0d4ec954fc4034b/.tmp/B/9f94221481804f42b846e9e9614916f6 as hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/697cd6a363e514bff0d4ec954fc4034b/B/9f94221481804f42b846e9e9614916f6 2024-11-16T09:59:59,795 INFO [RS:0;037c58a19ddb:37267-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 697cd6a363e514bff0d4ec954fc4034b/B of 697cd6a363e514bff0d4ec954fc4034b into 9f94221481804f42b846e9e9614916f6(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-16T09:59:59,795 DEBUG [RS:0;037c58a19ddb:37267-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 697cd6a363e514bff0d4ec954fc4034b: 2024-11-16T09:59:59,795 DEBUG [RS:0;037c58a19ddb:37267-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/697cd6a363e514bff0d4ec954fc4034b/.tmp/A/b0882ffe2ec9422087f061171d91e3bc as hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/697cd6a363e514bff0d4ec954fc4034b/A/b0882ffe2ec9422087f061171d91e3bc 2024-11-16T09:59:59,795 INFO [RS:0;037c58a19ddb:37267-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731751192181.697cd6a363e514bff0d4ec954fc4034b., storeName=697cd6a363e514bff0d4ec954fc4034b/B, priority=13, startTime=1731751199358; duration=0sec 2024-11-16T09:59:59,795 DEBUG [RS:0;037c58a19ddb:37267-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-16T09:59:59,795 DEBUG [RS:0;037c58a19ddb:37267-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 697cd6a363e514bff0d4ec954fc4034b:B 2024-11-16T09:59:59,795 DEBUG [RS:0;037c58a19ddb:37267-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-16T09:59:59,796 DEBUG [RS:0;037c58a19ddb:37267-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36106 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-16T09:59:59,796 DEBUG [RS:0;037c58a19ddb:37267-longCompactions-0 {}] regionserver.HStore(1541): 697cd6a363e514bff0d4ec954fc4034b/C is initiating minor compaction (all files) 2024-11-16T09:59:59,796 INFO [RS:0;037c58a19ddb:37267-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 697cd6a363e514bff0d4ec954fc4034b/C in TestAcidGuarantees,,1731751192181.697cd6a363e514bff0d4ec954fc4034b. 2024-11-16T09:59:59,796 INFO [RS:0;037c58a19ddb:37267-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/697cd6a363e514bff0d4ec954fc4034b/C/2d47a6a059304795ba891d2763d74fad, hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/697cd6a363e514bff0d4ec954fc4034b/C/6776f39b530c4e44bc12e7f8fc2004fc, hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/697cd6a363e514bff0d4ec954fc4034b/C/37cbfa8ef4e44d8da5b85d7a0ce5089d] into tmpdir=hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/697cd6a363e514bff0d4ec954fc4034b/.tmp, totalSize=35.3 K 2024-11-16T09:59:59,797 DEBUG [RS:0;037c58a19ddb:37267-longCompactions-0 {}] compactions.Compactor(225): Compacting 2d47a6a059304795ba891d2763d74fad, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=48, earliestPutTs=1731751196656 2024-11-16T09:59:59,797 DEBUG [RS:0;037c58a19ddb:37267-longCompactions-0 {}] compactions.Compactor(225): Compacting 6776f39b530c4e44bc12e7f8fc2004fc, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=76, earliestPutTs=1731751196708 2024-11-16T09:59:59,798 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/697cd6a363e514bff0d4ec954fc4034b/.tmp/C/e2fa6223e0924717854b53c074fd39fd is 50, key is test_row_0/C:col10/1731751199022/Put/seqid=0 2024-11-16T09:59:59,800 DEBUG [RS:0;037c58a19ddb:37267-longCompactions-0 {}] compactions.Compactor(225): Compacting 37cbfa8ef4e44d8da5b85d7a0ce5089d, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=88, earliestPutTs=1731751198888 2024-11-16T09:59:59,803 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37591 is added to blk_1073742290_1466 (size=12001) 2024-11-16T09:59:59,806 INFO [RS:0;037c58a19ddb:37267-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 697cd6a363e514bff0d4ec954fc4034b/A of 697cd6a363e514bff0d4ec954fc4034b into b0882ffe2ec9422087f061171d91e3bc(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-16T09:59:59,806 DEBUG [RS:0;037c58a19ddb:37267-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 697cd6a363e514bff0d4ec954fc4034b: 2024-11-16T09:59:59,806 INFO [RS:0;037c58a19ddb:37267-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731751192181.697cd6a363e514bff0d4ec954fc4034b., storeName=697cd6a363e514bff0d4ec954fc4034b/A, priority=13, startTime=1731751199358; duration=0sec 2024-11-16T09:59:59,806 DEBUG [RS:0;037c58a19ddb:37267-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-16T09:59:59,806 DEBUG [RS:0;037c58a19ddb:37267-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 697cd6a363e514bff0d4ec954fc4034b:A 2024-11-16T09:59:59,808 INFO [RS:0;037c58a19ddb:37267-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 697cd6a363e514bff0d4ec954fc4034b#C#compaction#393 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-16T09:59:59,808 DEBUG [RS:0;037c58a19ddb:37267-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/697cd6a363e514bff0d4ec954fc4034b/.tmp/C/1ca86ee061304d5baf51f01fb8c60b7b is 50, key is test_row_0/C:col10/1731751198913/Put/seqid=0 2024-11-16T09:59:59,811 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37591 is added to blk_1073742291_1467 (size=12207) 2024-11-16T09:59:59,816 DEBUG [RS:0;037c58a19ddb:37267-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/697cd6a363e514bff0d4ec954fc4034b/.tmp/C/1ca86ee061304d5baf51f01fb8c60b7b as hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/697cd6a363e514bff0d4ec954fc4034b/C/1ca86ee061304d5baf51f01fb8c60b7b 2024-11-16T09:59:59,820 INFO [RS:0;037c58a19ddb:37267-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 697cd6a363e514bff0d4ec954fc4034b/C of 697cd6a363e514bff0d4ec954fc4034b into 1ca86ee061304d5baf51f01fb8c60b7b(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-16T09:59:59,820 DEBUG [RS:0;037c58a19ddb:37267-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 697cd6a363e514bff0d4ec954fc4034b: 2024-11-16T09:59:59,820 INFO [RS:0;037c58a19ddb:37267-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731751192181.697cd6a363e514bff0d4ec954fc4034b., storeName=697cd6a363e514bff0d4ec954fc4034b/C, priority=13, startTime=1731751199358; duration=0sec 2024-11-16T09:59:59,820 DEBUG [RS:0;037c58a19ddb:37267-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-16T09:59:59,820 DEBUG [RS:0;037c58a19ddb:37267-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 697cd6a363e514bff0d4ec954fc4034b:C 2024-11-16T10:00:00,028 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=697cd6a363e514bff0d4ec954fc4034b, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T10:00:00,028 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36012 deadline: 1731751260023, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=697cd6a363e514bff0d4ec954fc4034b, server=037c58a19ddb,37267,1731751077844 2024-11-16T10:00:00,028 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=697cd6a363e514bff0d4ec954fc4034b, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T10:00:00,028 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36022 deadline: 1731751260023, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=697cd6a363e514bff0d4ec954fc4034b, server=037c58a19ddb,37267,1731751077844 2024-11-16T10:00:00,028 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731751192181.697cd6a363e514bff0d4ec954fc4034b., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 , the old value is region=TestAcidGuarantees,,1731751192181.697cd6a363e514bff0d4ec954fc4034b., hostname=037c58a19ddb,37267,1731751077844, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=697cd6a363e514bff0d4ec954fc4034b, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:00:00,028 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731751192181.697cd6a363e514bff0d4ec954fc4034b., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=697cd6a363e514bff0d4ec954fc4034b, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:00:00,028 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731751192181.697cd6a363e514bff0d4ec954fc4034b., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 because the exception is null or not the one we care about 2024-11-16T10:00:00,028 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731751192181.697cd6a363e514bff0d4ec954fc4034b., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 , the old value is region=TestAcidGuarantees,,1731751192181.697cd6a363e514bff0d4ec954fc4034b., hostname=037c58a19ddb,37267,1731751077844, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=697cd6a363e514bff0d4ec954fc4034b, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:00:00,028 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731751192181.697cd6a363e514bff0d4ec954fc4034b., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=697cd6a363e514bff0d4ec954fc4034b, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:00:00,028 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731751192181.697cd6a363e514bff0d4ec954fc4034b., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 because the exception is null or not the one we care about 2024-11-16T10:00:00,046 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=697cd6a363e514bff0d4ec954fc4034b, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T10:00:00,046 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36058 deadline: 1731751260042, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=697cd6a363e514bff0d4ec954fc4034b, server=037c58a19ddb,37267,1731751077844 2024-11-16T10:00:00,046 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731751192181.697cd6a363e514bff0d4ec954fc4034b., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 , the old value is region=TestAcidGuarantees,,1731751192181.697cd6a363e514bff0d4ec954fc4034b., hostname=037c58a19ddb,37267,1731751077844, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=697cd6a363e514bff0d4ec954fc4034b, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:00:00,046 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731751192181.697cd6a363e514bff0d4ec954fc4034b., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=697cd6a363e514bff0d4ec954fc4034b, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:00:00,046 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731751192181.697cd6a363e514bff0d4ec954fc4034b., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 because the exception is null or not the one we care about 2024-11-16T10:00:00,203 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=113 (bloomFilter=true), to=hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/697cd6a363e514bff0d4ec954fc4034b/.tmp/C/e2fa6223e0924717854b53c074fd39fd 2024-11-16T10:00:00,207 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/697cd6a363e514bff0d4ec954fc4034b/.tmp/A/93f09bcb15ca453ba30359031b06d38d as hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/697cd6a363e514bff0d4ec954fc4034b/A/93f09bcb15ca453ba30359031b06d38d 2024-11-16T10:00:00,210 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/697cd6a363e514bff0d4ec954fc4034b/A/93f09bcb15ca453ba30359031b06d38d, entries=200, sequenceid=113, filesize=14.0 K 2024-11-16T10:00:00,211 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/697cd6a363e514bff0d4ec954fc4034b/.tmp/B/34019d6dca5944bb901ff94f64922d8a as hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/697cd6a363e514bff0d4ec954fc4034b/B/34019d6dca5944bb901ff94f64922d8a 2024-11-16T10:00:00,214 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/697cd6a363e514bff0d4ec954fc4034b/B/34019d6dca5944bb901ff94f64922d8a, entries=150, sequenceid=113, filesize=11.7 K 2024-11-16T10:00:00,214 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/697cd6a363e514bff0d4ec954fc4034b/.tmp/C/e2fa6223e0924717854b53c074fd39fd as hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/697cd6a363e514bff0d4ec954fc4034b/C/e2fa6223e0924717854b53c074fd39fd 2024-11-16T10:00:00,217 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/697cd6a363e514bff0d4ec954fc4034b/C/e2fa6223e0924717854b53c074fd39fd, entries=150, sequenceid=113, filesize=11.7 K 2024-11-16T10:00:00,218 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~147.60 KB/151140, heapSize ~387.42 KB/396720, currentSize=53.67 KB/54960 for 697cd6a363e514bff0d4ec954fc4034b in 857ms, sequenceid=113, compaction requested=false 2024-11-16T10:00:00,218 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 697cd6a363e514bff0d4ec954fc4034b: 2024-11-16T10:00:00,536 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] regionserver.HRegion(8855): Flush requested on 697cd6a363e514bff0d4ec954fc4034b 2024-11-16T10:00:00,536 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-16T10:00:00,536 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 697cd6a363e514bff0d4ec954fc4034b 3/3 column families, dataSize=60.38 KB heapSize=158.95 KB 2024-11-16T10:00:00,541 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/697cd6a363e514bff0d4ec954fc4034b/.tmp/A/0e15f225c9fc496c9feeb6c7cfeaf92b is 50, key is test_row_1/A:col10/1731751200535/Put/seqid=0 2024-11-16T10:00:00,546 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37591 is added to blk_1073742292_1468 (size=9657) 2024-11-16T10:00:00,546 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=128 (bloomFilter=true), to=hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/697cd6a363e514bff0d4ec954fc4034b/.tmp/A/0e15f225c9fc496c9feeb6c7cfeaf92b 2024-11-16T10:00:00,553 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/697cd6a363e514bff0d4ec954fc4034b/.tmp/B/380b382ef5cb45f595d4ec3ad8d26436 is 50, key is test_row_1/B:col10/1731751200535/Put/seqid=0 2024-11-16T10:00:00,558 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37591 is added to blk_1073742293_1469 (size=9657) 2024-11-16T10:00:00,614 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=697cd6a363e514bff0d4ec954fc4034b, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T10:00:00,614 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36022 deadline: 1731751260605, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=697cd6a363e514bff0d4ec954fc4034b, server=037c58a19ddb,37267,1731751077844 2024-11-16T10:00:00,615 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=697cd6a363e514bff0d4ec954fc4034b, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T10:00:00,615 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731751192181.697cd6a363e514bff0d4ec954fc4034b., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 , the old value is region=TestAcidGuarantees,,1731751192181.697cd6a363e514bff0d4ec954fc4034b., hostname=037c58a19ddb,37267,1731751077844, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=697cd6a363e514bff0d4ec954fc4034b, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:00:00,615 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731751192181.697cd6a363e514bff0d4ec954fc4034b., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=697cd6a363e514bff0d4ec954fc4034b, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:00:00,615 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36058 deadline: 1731751260605, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=697cd6a363e514bff0d4ec954fc4034b, server=037c58a19ddb,37267,1731751077844 2024-11-16T10:00:00,615 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731751192181.697cd6a363e514bff0d4ec954fc4034b., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 because the exception is null or not the one we care about 2024-11-16T10:00:00,615 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731751192181.697cd6a363e514bff0d4ec954fc4034b., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 , the old value is region=TestAcidGuarantees,,1731751192181.697cd6a363e514bff0d4ec954fc4034b., hostname=037c58a19ddb,37267,1731751077844, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=697cd6a363e514bff0d4ec954fc4034b, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:00:00,615 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731751192181.697cd6a363e514bff0d4ec954fc4034b., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=697cd6a363e514bff0d4ec954fc4034b, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:00:00,615 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731751192181.697cd6a363e514bff0d4ec954fc4034b., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 because the exception is null or not the one we care about 2024-11-16T10:00:00,620 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=697cd6a363e514bff0d4ec954fc4034b, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T10:00:00,620 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36012 deadline: 1731751260611, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=697cd6a363e514bff0d4ec954fc4034b, server=037c58a19ddb,37267,1731751077844 2024-11-16T10:00:00,620 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731751192181.697cd6a363e514bff0d4ec954fc4034b., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 , the old value is region=TestAcidGuarantees,,1731751192181.697cd6a363e514bff0d4ec954fc4034b., hostname=037c58a19ddb,37267,1731751077844, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=697cd6a363e514bff0d4ec954fc4034b, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:00:00,620 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731751192181.697cd6a363e514bff0d4ec954fc4034b., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=697cd6a363e514bff0d4ec954fc4034b, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:00:00,620 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731751192181.697cd6a363e514bff0d4ec954fc4034b., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 because the exception is null or not the one we care about 2024-11-16T10:00:00,659 WARN [regionserver/037c58a19ddb:0.Chore.1 {}] hbase.ExecutorStatusChore(69): RS_COMPACTED_FILES_DISCHARGER's size info, queued: 2, running: 1 2024-11-16T10:00:00,659 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/037c58a19ddb:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] regionserver.HStore(2317): Moving the files [hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/697cd6a363e514bff0d4ec954fc4034b/A/656d8bd3c1f740f590bef993761e4820, hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/697cd6a363e514bff0d4ec954fc4034b/A/aedcfb03de7247b49de87f00d96a1a32, hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/697cd6a363e514bff0d4ec954fc4034b/A/38f40b75e4b343319edd57d2aca67982, hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/697cd6a363e514bff0d4ec954fc4034b/A/824fa0f6583148b1a09539c16b299178, hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/697cd6a363e514bff0d4ec954fc4034b/A/fbfadead6dcb499097ec150548f8c3a1, hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/697cd6a363e514bff0d4ec954fc4034b/A/693c6bb535ea4f67b3213cf70d77313d] to archive 2024-11-16T10:00:00,660 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/037c58a19ddb:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-16T10:00:00,662 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/037c58a19ddb:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/697cd6a363e514bff0d4ec954fc4034b/A/656d8bd3c1f740f590bef993761e4820 to hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/archive/data/default/TestAcidGuarantees/697cd6a363e514bff0d4ec954fc4034b/A/656d8bd3c1f740f590bef993761e4820 2024-11-16T10:00:00,663 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/037c58a19ddb:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/697cd6a363e514bff0d4ec954fc4034b/A/aedcfb03de7247b49de87f00d96a1a32 to hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/archive/data/default/TestAcidGuarantees/697cd6a363e514bff0d4ec954fc4034b/A/aedcfb03de7247b49de87f00d96a1a32 2024-11-16T10:00:00,664 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/037c58a19ddb:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/697cd6a363e514bff0d4ec954fc4034b/A/38f40b75e4b343319edd57d2aca67982 to hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/archive/data/default/TestAcidGuarantees/697cd6a363e514bff0d4ec954fc4034b/A/38f40b75e4b343319edd57d2aca67982 2024-11-16T10:00:00,665 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/037c58a19ddb:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/697cd6a363e514bff0d4ec954fc4034b/A/824fa0f6583148b1a09539c16b299178 to hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/archive/data/default/TestAcidGuarantees/697cd6a363e514bff0d4ec954fc4034b/A/824fa0f6583148b1a09539c16b299178 2024-11-16T10:00:00,666 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/037c58a19ddb:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/697cd6a363e514bff0d4ec954fc4034b/A/fbfadead6dcb499097ec150548f8c3a1 to hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/archive/data/default/TestAcidGuarantees/697cd6a363e514bff0d4ec954fc4034b/A/fbfadead6dcb499097ec150548f8c3a1 2024-11-16T10:00:00,667 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/037c58a19ddb:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/697cd6a363e514bff0d4ec954fc4034b/A/693c6bb535ea4f67b3213cf70d77313d to hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/archive/data/default/TestAcidGuarantees/697cd6a363e514bff0d4ec954fc4034b/A/693c6bb535ea4f67b3213cf70d77313d 2024-11-16T10:00:00,668 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/037c58a19ddb:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] regionserver.HStore(2317): Moving the files [hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/697cd6a363e514bff0d4ec954fc4034b/B/b879c013fa994e78b1fe7c6327ecd121, hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/697cd6a363e514bff0d4ec954fc4034b/B/45b14ada2cc046159725de6299e3026e, hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/697cd6a363e514bff0d4ec954fc4034b/B/20ca0a467e464862a5d2d9fef6e69a82, hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/697cd6a363e514bff0d4ec954fc4034b/B/19d2ff883d9a4216a9c2e142390c211d, hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/697cd6a363e514bff0d4ec954fc4034b/B/e9af6ad3179a4ff38bce9dafdfed880b, hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/697cd6a363e514bff0d4ec954fc4034b/B/0096a10bcf1b406dba763c0e979e5c5e] to archive 2024-11-16T10:00:00,668 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/037c58a19ddb:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-16T10:00:00,670 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/037c58a19ddb:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/697cd6a363e514bff0d4ec954fc4034b/B/b879c013fa994e78b1fe7c6327ecd121 to hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/archive/data/default/TestAcidGuarantees/697cd6a363e514bff0d4ec954fc4034b/B/b879c013fa994e78b1fe7c6327ecd121 2024-11-16T10:00:00,671 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/037c58a19ddb:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/697cd6a363e514bff0d4ec954fc4034b/B/45b14ada2cc046159725de6299e3026e to hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/archive/data/default/TestAcidGuarantees/697cd6a363e514bff0d4ec954fc4034b/B/45b14ada2cc046159725de6299e3026e 2024-11-16T10:00:00,672 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/037c58a19ddb:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/697cd6a363e514bff0d4ec954fc4034b/B/20ca0a467e464862a5d2d9fef6e69a82 to hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/archive/data/default/TestAcidGuarantees/697cd6a363e514bff0d4ec954fc4034b/B/20ca0a467e464862a5d2d9fef6e69a82 2024-11-16T10:00:00,672 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/037c58a19ddb:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/697cd6a363e514bff0d4ec954fc4034b/B/19d2ff883d9a4216a9c2e142390c211d to hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/archive/data/default/TestAcidGuarantees/697cd6a363e514bff0d4ec954fc4034b/B/19d2ff883d9a4216a9c2e142390c211d 2024-11-16T10:00:00,673 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/037c58a19ddb:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/697cd6a363e514bff0d4ec954fc4034b/B/e9af6ad3179a4ff38bce9dafdfed880b to hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/archive/data/default/TestAcidGuarantees/697cd6a363e514bff0d4ec954fc4034b/B/e9af6ad3179a4ff38bce9dafdfed880b 2024-11-16T10:00:00,674 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/037c58a19ddb:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/697cd6a363e514bff0d4ec954fc4034b/B/0096a10bcf1b406dba763c0e979e5c5e to hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/archive/data/default/TestAcidGuarantees/697cd6a363e514bff0d4ec954fc4034b/B/0096a10bcf1b406dba763c0e979e5c5e 2024-11-16T10:00:00,676 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/037c58a19ddb:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] regionserver.HStore(2317): Moving the files [hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/697cd6a363e514bff0d4ec954fc4034b/C/a5314ebc7f464faba37fa36b02402253, hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/697cd6a363e514bff0d4ec954fc4034b/C/1180197cb1f34930b834b28b614575b2, hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/697cd6a363e514bff0d4ec954fc4034b/C/2d47a6a059304795ba891d2763d74fad, hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/697cd6a363e514bff0d4ec954fc4034b/C/a7411494c98043698e28066a2f4b930b, hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/697cd6a363e514bff0d4ec954fc4034b/C/6776f39b530c4e44bc12e7f8fc2004fc, hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/697cd6a363e514bff0d4ec954fc4034b/C/37cbfa8ef4e44d8da5b85d7a0ce5089d] to archive 2024-11-16T10:00:00,676 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/037c58a19ddb:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-16T10:00:00,678 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/037c58a19ddb:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/697cd6a363e514bff0d4ec954fc4034b/C/a5314ebc7f464faba37fa36b02402253 to hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/archive/data/default/TestAcidGuarantees/697cd6a363e514bff0d4ec954fc4034b/C/a5314ebc7f464faba37fa36b02402253 2024-11-16T10:00:00,678 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/037c58a19ddb:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/697cd6a363e514bff0d4ec954fc4034b/C/1180197cb1f34930b834b28b614575b2 to hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/archive/data/default/TestAcidGuarantees/697cd6a363e514bff0d4ec954fc4034b/C/1180197cb1f34930b834b28b614575b2 2024-11-16T10:00:00,679 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/037c58a19ddb:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/697cd6a363e514bff0d4ec954fc4034b/C/2d47a6a059304795ba891d2763d74fad to hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/archive/data/default/TestAcidGuarantees/697cd6a363e514bff0d4ec954fc4034b/C/2d47a6a059304795ba891d2763d74fad 2024-11-16T10:00:00,680 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/037c58a19ddb:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/697cd6a363e514bff0d4ec954fc4034b/C/a7411494c98043698e28066a2f4b930b to hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/archive/data/default/TestAcidGuarantees/697cd6a363e514bff0d4ec954fc4034b/C/a7411494c98043698e28066a2f4b930b 2024-11-16T10:00:00,681 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/037c58a19ddb:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/697cd6a363e514bff0d4ec954fc4034b/C/6776f39b530c4e44bc12e7f8fc2004fc to hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/archive/data/default/TestAcidGuarantees/697cd6a363e514bff0d4ec954fc4034b/C/6776f39b530c4e44bc12e7f8fc2004fc 2024-11-16T10:00:00,682 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/037c58a19ddb:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/697cd6a363e514bff0d4ec954fc4034b/C/37cbfa8ef4e44d8da5b85d7a0ce5089d to hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/archive/data/default/TestAcidGuarantees/697cd6a363e514bff0d4ec954fc4034b/C/37cbfa8ef4e44d8da5b85d7a0ce5089d 2024-11-16T10:00:00,683 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37877 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=121 2024-11-16T10:00:00,683 INFO [RPCClient-NioEventLoopGroup-4-10 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-16T10:00:00,684 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37877 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-16T10:00:00,684 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37877 {}] procedure2.ProcedureExecutor(1139): Stored pid=123, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=123, table=TestAcidGuarantees 2024-11-16T10:00:00,685 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37877 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=123 2024-11-16T10:00:00,685 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=123, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=123, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-16T10:00:00,686 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=123, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=123, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-16T10:00:00,686 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=124, ppid=123, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-16T10:00:00,728 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=697cd6a363e514bff0d4ec954fc4034b, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T10:00:00,729 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36022 deadline: 1731751260723, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=697cd6a363e514bff0d4ec954fc4034b, server=037c58a19ddb,37267,1731751077844 2024-11-16T10:00:00,729 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=697cd6a363e514bff0d4ec954fc4034b, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T10:00:00,729 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36058 deadline: 1731751260723, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=697cd6a363e514bff0d4ec954fc4034b, server=037c58a19ddb,37267,1731751077844 2024-11-16T10:00:00,729 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731751192181.697cd6a363e514bff0d4ec954fc4034b., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 , the old value is region=TestAcidGuarantees,,1731751192181.697cd6a363e514bff0d4ec954fc4034b., hostname=037c58a19ddb,37267,1731751077844, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=697cd6a363e514bff0d4ec954fc4034b, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:00:00,729 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731751192181.697cd6a363e514bff0d4ec954fc4034b., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=697cd6a363e514bff0d4ec954fc4034b, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:00:00,729 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731751192181.697cd6a363e514bff0d4ec954fc4034b., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 because the exception is null or not the one we care about 2024-11-16T10:00:00,729 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=697cd6a363e514bff0d4ec954fc4034b, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T10:00:00,729 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731751192181.697cd6a363e514bff0d4ec954fc4034b., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 , the old value is region=TestAcidGuarantees,,1731751192181.697cd6a363e514bff0d4ec954fc4034b., hostname=037c58a19ddb,37267,1731751077844, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=697cd6a363e514bff0d4ec954fc4034b, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:00:00,729 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731751192181.697cd6a363e514bff0d4ec954fc4034b., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=697cd6a363e514bff0d4ec954fc4034b, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:00:00,729 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731751192181.697cd6a363e514bff0d4ec954fc4034b., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 because the exception is null or not the one we care about 2024-11-16T10:00:00,729 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36012 deadline: 1731751260723, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=697cd6a363e514bff0d4ec954fc4034b, server=037c58a19ddb,37267,1731751077844 2024-11-16T10:00:00,729 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731751192181.697cd6a363e514bff0d4ec954fc4034b., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 , the old value is region=TestAcidGuarantees,,1731751192181.697cd6a363e514bff0d4ec954fc4034b., hostname=037c58a19ddb,37267,1731751077844, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=697cd6a363e514bff0d4ec954fc4034b, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:00:00,729 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731751192181.697cd6a363e514bff0d4ec954fc4034b., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=697cd6a363e514bff0d4ec954fc4034b, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:00:00,729 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731751192181.697cd6a363e514bff0d4ec954fc4034b., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 because the exception is null or not the one we care about 2024-11-16T10:00:00,793 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37877 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=123 2024-11-16T10:00:00,838 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=37267 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=124 2024-11-16T10:00:00,838 DEBUG [RS_FLUSH_OPERATIONS-regionserver/037c58a19ddb:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731751192181.697cd6a363e514bff0d4ec954fc4034b. 2024-11-16T10:00:00,839 DEBUG [RS_FLUSH_OPERATIONS-regionserver/037c58a19ddb:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731751192181.697cd6a363e514bff0d4ec954fc4034b. as already flushing 2024-11-16T10:00:00,839 DEBUG [RS_FLUSH_OPERATIONS-regionserver/037c58a19ddb:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731751192181.697cd6a363e514bff0d4ec954fc4034b. 2024-11-16T10:00:00,839 ERROR [RS_FLUSH_OPERATIONS-regionserver/037c58a19ddb:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] handler.RSProcedureHandler(58): pid=124 java.io.IOException: Unable to complete flush {ENCODED => 697cd6a363e514bff0d4ec954fc4034b, NAME => 'TestAcidGuarantees,,1731751192181.697cd6a363e514bff0d4ec954fc4034b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T10:00:00,839 DEBUG [RS_FLUSH_OPERATIONS-regionserver/037c58a19ddb:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=124 java.io.IOException: Unable to complete flush {ENCODED => 697cd6a363e514bff0d4ec954fc4034b, NAME => 'TestAcidGuarantees,,1731751192181.697cd6a363e514bff0d4ec954fc4034b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T10:00:00,840 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37877 {}] master.HMaster(4177): Remote procedure failed, pid=124 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 697cd6a363e514bff0d4ec954fc4034b, NAME => 'TestAcidGuarantees,,1731751192181.697cd6a363e514bff0d4ec954fc4034b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 697cd6a363e514bff0d4ec954fc4034b, NAME => 'TestAcidGuarantees,,1731751192181.697cd6a363e514bff0d4ec954fc4034b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T10:00:00,929 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=697cd6a363e514bff0d4ec954fc4034b, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T10:00:00,929 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36086 deadline: 1731751260923, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=697cd6a363e514bff0d4ec954fc4034b, server=037c58a19ddb,37267,1731751077844 2024-11-16T10:00:00,929 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=697cd6a363e514bff0d4ec954fc4034b, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T10:00:00,929 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36046 deadline: 1731751260923, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=697cd6a363e514bff0d4ec954fc4034b, server=037c58a19ddb,37267,1731751077844 2024-11-16T10:00:00,929 WARN [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRpcRetryingCaller(177): Call to 037c58a19ddb,37267,1731751077844 for 'test_row_2' in 697cd6a363e514bff0d4ec954fc4034b of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4228 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=697cd6a363e514bff0d4ec954fc4034b, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor44.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=697cd6a363e514bff0d4ec954fc4034b, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-16T10:00:00,929 WARN [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRpcRetryingCaller(177): Call to 037c58a19ddb,37267,1731751077844 for 'test_row_2' in 697cd6a363e514bff0d4ec954fc4034b of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4216 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=697cd6a363e514bff0d4ec954fc4034b, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor44.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=697cd6a363e514bff0d4ec954fc4034b, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-16T10:00:00,930 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731751192181.697cd6a363e514bff0d4ec954fc4034b., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 , the old value is region=TestAcidGuarantees,,1731751192181.697cd6a363e514bff0d4ec954fc4034b., hostname=037c58a19ddb,37267,1731751077844, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=697cd6a363e514bff0d4ec954fc4034b, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:00:00,930 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731751192181.697cd6a363e514bff0d4ec954fc4034b., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 , the old value is region=TestAcidGuarantees,,1731751192181.697cd6a363e514bff0d4ec954fc4034b., hostname=037c58a19ddb,37267,1731751077844, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=697cd6a363e514bff0d4ec954fc4034b, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:00:00,930 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731751192181.697cd6a363e514bff0d4ec954fc4034b., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=697cd6a363e514bff0d4ec954fc4034b, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:00:00,930 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731751192181.697cd6a363e514bff0d4ec954fc4034b., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 because the exception is null or not the one we care about 2024-11-16T10:00:00,930 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731751192181.697cd6a363e514bff0d4ec954fc4034b., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=697cd6a363e514bff0d4ec954fc4034b, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:00:00,930 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731751192181.697cd6a363e514bff0d4ec954fc4034b., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 because the exception is null or not the one we care about 2024-11-16T10:00:00,938 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=697cd6a363e514bff0d4ec954fc4034b, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T10:00:00,938 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36022 deadline: 1731751260933, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=697cd6a363e514bff0d4ec954fc4034b, server=037c58a19ddb,37267,1731751077844 2024-11-16T10:00:00,938 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=697cd6a363e514bff0d4ec954fc4034b, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T10:00:00,938 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731751192181.697cd6a363e514bff0d4ec954fc4034b., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 , the old value is region=TestAcidGuarantees,,1731751192181.697cd6a363e514bff0d4ec954fc4034b., hostname=037c58a19ddb,37267,1731751077844, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=697cd6a363e514bff0d4ec954fc4034b, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:00:00,938 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731751192181.697cd6a363e514bff0d4ec954fc4034b., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=697cd6a363e514bff0d4ec954fc4034b, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:00:00,938 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731751192181.697cd6a363e514bff0d4ec954fc4034b., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 because the exception is null or not the one we care about 2024-11-16T10:00:00,938 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36058 deadline: 1731751260933, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=697cd6a363e514bff0d4ec954fc4034b, server=037c58a19ddb,37267,1731751077844 2024-11-16T10:00:00,939 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731751192181.697cd6a363e514bff0d4ec954fc4034b., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 , the old value is region=TestAcidGuarantees,,1731751192181.697cd6a363e514bff0d4ec954fc4034b., hostname=037c58a19ddb,37267,1731751077844, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=697cd6a363e514bff0d4ec954fc4034b, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:00:00,939 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731751192181.697cd6a363e514bff0d4ec954fc4034b., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=697cd6a363e514bff0d4ec954fc4034b, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:00:00,939 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731751192181.697cd6a363e514bff0d4ec954fc4034b., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 because the exception is null or not the one we care about 2024-11-16T10:00:00,939 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=697cd6a363e514bff0d4ec954fc4034b, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T10:00:00,939 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36012 deadline: 1731751260933, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=697cd6a363e514bff0d4ec954fc4034b, server=037c58a19ddb,37267,1731751077844 2024-11-16T10:00:00,939 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731751192181.697cd6a363e514bff0d4ec954fc4034b., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 , the old value is region=TestAcidGuarantees,,1731751192181.697cd6a363e514bff0d4ec954fc4034b., hostname=037c58a19ddb,37267,1731751077844, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=697cd6a363e514bff0d4ec954fc4034b, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:00:00,939 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731751192181.697cd6a363e514bff0d4ec954fc4034b., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=697cd6a363e514bff0d4ec954fc4034b, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:00:00,939 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731751192181.697cd6a363e514bff0d4ec954fc4034b., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 because the exception is null or not the one we care about 2024-11-16T10:00:00,959 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=128 (bloomFilter=true), to=hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/697cd6a363e514bff0d4ec954fc4034b/.tmp/B/380b382ef5cb45f595d4ec3ad8d26436 2024-11-16T10:00:00,965 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/697cd6a363e514bff0d4ec954fc4034b/.tmp/C/9b994a5406404b3fa89fea7191ed65e7 is 50, key is test_row_1/C:col10/1731751200535/Put/seqid=0 2024-11-16T10:00:00,969 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37591 is added to blk_1073742294_1470 (size=9657) 2024-11-16T10:00:00,991 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=37267 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=124 2024-11-16T10:00:00,992 DEBUG [RS_FLUSH_OPERATIONS-regionserver/037c58a19ddb:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731751192181.697cd6a363e514bff0d4ec954fc4034b. 2024-11-16T10:00:00,992 DEBUG [RS_FLUSH_OPERATIONS-regionserver/037c58a19ddb:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731751192181.697cd6a363e514bff0d4ec954fc4034b. as already flushing 2024-11-16T10:00:00,992 DEBUG [RS_FLUSH_OPERATIONS-regionserver/037c58a19ddb:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731751192181.697cd6a363e514bff0d4ec954fc4034b. 2024-11-16T10:00:00,992 ERROR [RS_FLUSH_OPERATIONS-regionserver/037c58a19ddb:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] handler.RSProcedureHandler(58): pid=124 java.io.IOException: Unable to complete flush {ENCODED => 697cd6a363e514bff0d4ec954fc4034b, NAME => 'TestAcidGuarantees,,1731751192181.697cd6a363e514bff0d4ec954fc4034b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T10:00:00,992 DEBUG [RS_FLUSH_OPERATIONS-regionserver/037c58a19ddb:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=124 java.io.IOException: Unable to complete flush {ENCODED => 697cd6a363e514bff0d4ec954fc4034b, NAME => 'TestAcidGuarantees,,1731751192181.697cd6a363e514bff0d4ec954fc4034b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T10:00:00,992 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37877 {}] master.HMaster(4177): Remote procedure failed, pid=124 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 697cd6a363e514bff0d4ec954fc4034b, NAME => 'TestAcidGuarantees,,1731751192181.697cd6a363e514bff0d4ec954fc4034b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 697cd6a363e514bff0d4ec954fc4034b, NAME => 'TestAcidGuarantees,,1731751192181.697cd6a363e514bff0d4ec954fc4034b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T10:00:01,003 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37877 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=123 2024-11-16T10:00:01,144 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=37267 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=124 2024-11-16T10:00:01,144 DEBUG [RS_FLUSH_OPERATIONS-regionserver/037c58a19ddb:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731751192181.697cd6a363e514bff0d4ec954fc4034b. 2024-11-16T10:00:01,144 DEBUG [RS_FLUSH_OPERATIONS-regionserver/037c58a19ddb:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731751192181.697cd6a363e514bff0d4ec954fc4034b. as already flushing 2024-11-16T10:00:01,144 DEBUG [RS_FLUSH_OPERATIONS-regionserver/037c58a19ddb:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731751192181.697cd6a363e514bff0d4ec954fc4034b. 2024-11-16T10:00:01,144 ERROR [RS_FLUSH_OPERATIONS-regionserver/037c58a19ddb:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] handler.RSProcedureHandler(58): pid=124 java.io.IOException: Unable to complete flush {ENCODED => 697cd6a363e514bff0d4ec954fc4034b, NAME => 'TestAcidGuarantees,,1731751192181.697cd6a363e514bff0d4ec954fc4034b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T10:00:01,145 DEBUG [RS_FLUSH_OPERATIONS-regionserver/037c58a19ddb:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=124 java.io.IOException: Unable to complete flush {ENCODED => 697cd6a363e514bff0d4ec954fc4034b, NAME => 'TestAcidGuarantees,,1731751192181.697cd6a363e514bff0d4ec954fc4034b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T10:00:01,145 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37877 {}] master.HMaster(4177): Remote procedure failed, pid=124 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 697cd6a363e514bff0d4ec954fc4034b, NAME => 'TestAcidGuarantees,,1731751192181.697cd6a363e514bff0d4ec954fc4034b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 697cd6a363e514bff0d4ec954fc4034b, NAME => 'TestAcidGuarantees,,1731751192181.697cd6a363e514bff0d4ec954fc4034b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T10:00:01,244 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=697cd6a363e514bff0d4ec954fc4034b, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T10:00:01,245 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36022 deadline: 1731751261243, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=697cd6a363e514bff0d4ec954fc4034b, server=037c58a19ddb,37267,1731751077844 2024-11-16T10:00:01,245 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=697cd6a363e514bff0d4ec954fc4034b, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T10:00:01,245 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731751192181.697cd6a363e514bff0d4ec954fc4034b., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 , the old value is region=TestAcidGuarantees,,1731751192181.697cd6a363e514bff0d4ec954fc4034b., hostname=037c58a19ddb,37267,1731751077844, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=697cd6a363e514bff0d4ec954fc4034b, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:00:01,245 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36058 deadline: 1731751261243, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=697cd6a363e514bff0d4ec954fc4034b, server=037c58a19ddb,37267,1731751077844 2024-11-16T10:00:01,245 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731751192181.697cd6a363e514bff0d4ec954fc4034b., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=697cd6a363e514bff0d4ec954fc4034b, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:00:01,245 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731751192181.697cd6a363e514bff0d4ec954fc4034b., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 because the exception is null or not the one we care about 2024-11-16T10:00:01,245 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=697cd6a363e514bff0d4ec954fc4034b, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T10:00:01,245 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731751192181.697cd6a363e514bff0d4ec954fc4034b., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 , the old value is region=TestAcidGuarantees,,1731751192181.697cd6a363e514bff0d4ec954fc4034b., hostname=037c58a19ddb,37267,1731751077844, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=697cd6a363e514bff0d4ec954fc4034b, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:00:01,245 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731751192181.697cd6a363e514bff0d4ec954fc4034b., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=697cd6a363e514bff0d4ec954fc4034b, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:00:01,245 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36012 deadline: 1731751261243, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=697cd6a363e514bff0d4ec954fc4034b, server=037c58a19ddb,37267,1731751077844 2024-11-16T10:00:01,245 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731751192181.697cd6a363e514bff0d4ec954fc4034b., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 because the exception is null or not the one we care about 2024-11-16T10:00:01,246 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731751192181.697cd6a363e514bff0d4ec954fc4034b., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 , the old value is region=TestAcidGuarantees,,1731751192181.697cd6a363e514bff0d4ec954fc4034b., hostname=037c58a19ddb,37267,1731751077844, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=697cd6a363e514bff0d4ec954fc4034b, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:00:01,246 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731751192181.697cd6a363e514bff0d4ec954fc4034b., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=697cd6a363e514bff0d4ec954fc4034b, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:00:01,246 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731751192181.697cd6a363e514bff0d4ec954fc4034b., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 because the exception is null or not the one we care about 2024-11-16T10:00:01,297 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=37267 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=124 2024-11-16T10:00:01,297 DEBUG [RS_FLUSH_OPERATIONS-regionserver/037c58a19ddb:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731751192181.697cd6a363e514bff0d4ec954fc4034b. 2024-11-16T10:00:01,297 DEBUG [RS_FLUSH_OPERATIONS-regionserver/037c58a19ddb:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731751192181.697cd6a363e514bff0d4ec954fc4034b. as already flushing 2024-11-16T10:00:01,297 DEBUG [RS_FLUSH_OPERATIONS-regionserver/037c58a19ddb:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731751192181.697cd6a363e514bff0d4ec954fc4034b. 2024-11-16T10:00:01,297 ERROR [RS_FLUSH_OPERATIONS-regionserver/037c58a19ddb:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] handler.RSProcedureHandler(58): pid=124 java.io.IOException: Unable to complete flush {ENCODED => 697cd6a363e514bff0d4ec954fc4034b, NAME => 'TestAcidGuarantees,,1731751192181.697cd6a363e514bff0d4ec954fc4034b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T10:00:01,298 DEBUG [RS_FLUSH_OPERATIONS-regionserver/037c58a19ddb:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=124 java.io.IOException: Unable to complete flush {ENCODED => 697cd6a363e514bff0d4ec954fc4034b, NAME => 'TestAcidGuarantees,,1731751192181.697cd6a363e514bff0d4ec954fc4034b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T10:00:01,298 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37877 {}] master.HMaster(4177): Remote procedure failed, pid=124 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 697cd6a363e514bff0d4ec954fc4034b, NAME => 'TestAcidGuarantees,,1731751192181.697cd6a363e514bff0d4ec954fc4034b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 697cd6a363e514bff0d4ec954fc4034b, NAME => 'TestAcidGuarantees,,1731751192181.697cd6a363e514bff0d4ec954fc4034b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T10:00:01,312 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37877 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=123 2024-11-16T10:00:01,369 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=128 (bloomFilter=true), to=hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/697cd6a363e514bff0d4ec954fc4034b/.tmp/C/9b994a5406404b3fa89fea7191ed65e7 2024-11-16T10:00:01,373 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/697cd6a363e514bff0d4ec954fc4034b/.tmp/A/0e15f225c9fc496c9feeb6c7cfeaf92b as hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/697cd6a363e514bff0d4ec954fc4034b/A/0e15f225c9fc496c9feeb6c7cfeaf92b 2024-11-16T10:00:01,376 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/697cd6a363e514bff0d4ec954fc4034b/A/0e15f225c9fc496c9feeb6c7cfeaf92b, entries=100, sequenceid=128, filesize=9.4 K 2024-11-16T10:00:01,377 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/697cd6a363e514bff0d4ec954fc4034b/.tmp/B/380b382ef5cb45f595d4ec3ad8d26436 as hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/697cd6a363e514bff0d4ec954fc4034b/B/380b382ef5cb45f595d4ec3ad8d26436 2024-11-16T10:00:01,380 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/697cd6a363e514bff0d4ec954fc4034b/B/380b382ef5cb45f595d4ec3ad8d26436, entries=100, sequenceid=128, filesize=9.4 K 2024-11-16T10:00:01,380 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/697cd6a363e514bff0d4ec954fc4034b/.tmp/C/9b994a5406404b3fa89fea7191ed65e7 as hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/697cd6a363e514bff0d4ec954fc4034b/C/9b994a5406404b3fa89fea7191ed65e7 2024-11-16T10:00:01,384 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/697cd6a363e514bff0d4ec954fc4034b/C/9b994a5406404b3fa89fea7191ed65e7, entries=100, sequenceid=128, filesize=9.4 K 2024-11-16T10:00:01,385 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~158.91 KB/162720, currentSize=140.89 KB/144270 for 697cd6a363e514bff0d4ec954fc4034b in 849ms, sequenceid=128, compaction requested=true 2024-11-16T10:00:01,385 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 697cd6a363e514bff0d4ec954fc4034b: 2024-11-16T10:00:01,385 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 697cd6a363e514bff0d4ec954fc4034b:A, priority=-2147483648, current under compaction store size is 1 2024-11-16T10:00:01,385 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-16T10:00:01,385 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 697cd6a363e514bff0d4ec954fc4034b:B, priority=-2147483648, current under compaction store size is 2 2024-11-16T10:00:01,385 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-16T10:00:01,385 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 697cd6a363e514bff0d4ec954fc4034b:C, priority=-2147483648, current under compaction store size is 3 2024-11-16T10:00:01,385 DEBUG [RS:0;037c58a19ddb:37267-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-16T10:00:01,385 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-16T10:00:01,385 DEBUG [RS:0;037c58a19ddb:37267-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-16T10:00:01,386 DEBUG [RS:0;037c58a19ddb:37267-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 33865 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-16T10:00:01,386 DEBUG [RS:0;037c58a19ddb:37267-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36205 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-16T10:00:01,386 DEBUG [RS:0;037c58a19ddb:37267-shortCompactions-0 {}] regionserver.HStore(1541): 697cd6a363e514bff0d4ec954fc4034b/A is initiating minor compaction (all files) 2024-11-16T10:00:01,386 DEBUG [RS:0;037c58a19ddb:37267-longCompactions-0 {}] regionserver.HStore(1541): 697cd6a363e514bff0d4ec954fc4034b/B is initiating minor compaction (all files) 2024-11-16T10:00:01,386 INFO [RS:0;037c58a19ddb:37267-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 697cd6a363e514bff0d4ec954fc4034b/A in TestAcidGuarantees,,1731751192181.697cd6a363e514bff0d4ec954fc4034b. 2024-11-16T10:00:01,386 INFO [RS:0;037c58a19ddb:37267-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 697cd6a363e514bff0d4ec954fc4034b/B in TestAcidGuarantees,,1731751192181.697cd6a363e514bff0d4ec954fc4034b. 2024-11-16T10:00:01,386 INFO [RS:0;037c58a19ddb:37267-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/697cd6a363e514bff0d4ec954fc4034b/A/b0882ffe2ec9422087f061171d91e3bc, hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/697cd6a363e514bff0d4ec954fc4034b/A/93f09bcb15ca453ba30359031b06d38d, hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/697cd6a363e514bff0d4ec954fc4034b/A/0e15f225c9fc496c9feeb6c7cfeaf92b] into tmpdir=hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/697cd6a363e514bff0d4ec954fc4034b/.tmp, totalSize=35.4 K 2024-11-16T10:00:01,386 INFO [RS:0;037c58a19ddb:37267-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/697cd6a363e514bff0d4ec954fc4034b/B/9f94221481804f42b846e9e9614916f6, hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/697cd6a363e514bff0d4ec954fc4034b/B/34019d6dca5944bb901ff94f64922d8a, hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/697cd6a363e514bff0d4ec954fc4034b/B/380b382ef5cb45f595d4ec3ad8d26436] into tmpdir=hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/697cd6a363e514bff0d4ec954fc4034b/.tmp, totalSize=33.1 K 2024-11-16T10:00:01,387 DEBUG [RS:0;037c58a19ddb:37267-shortCompactions-0 {}] compactions.Compactor(225): Compacting b0882ffe2ec9422087f061171d91e3bc, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=88, earliestPutTs=1731751198888 2024-11-16T10:00:01,387 DEBUG [RS:0;037c58a19ddb:37267-longCompactions-0 {}] compactions.Compactor(225): Compacting 9f94221481804f42b846e9e9614916f6, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=88, earliestPutTs=1731751198888 2024-11-16T10:00:01,387 DEBUG [RS:0;037c58a19ddb:37267-longCompactions-0 {}] compactions.Compactor(225): Compacting 34019d6dca5944bb901ff94f64922d8a, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=113, earliestPutTs=1731751199022 2024-11-16T10:00:01,387 DEBUG [RS:0;037c58a19ddb:37267-shortCompactions-0 {}] compactions.Compactor(225): Compacting 93f09bcb15ca453ba30359031b06d38d, keycount=200, bloomtype=ROW, size=14.0 K, encoding=NONE, compression=NONE, seqNum=113, earliestPutTs=1731751199022 2024-11-16T10:00:01,387 DEBUG [RS:0;037c58a19ddb:37267-longCompactions-0 {}] compactions.Compactor(225): Compacting 380b382ef5cb45f595d4ec3ad8d26436, keycount=100, bloomtype=ROW, size=9.4 K, encoding=NONE, compression=NONE, seqNum=128, earliestPutTs=1731751199398 2024-11-16T10:00:01,387 DEBUG [RS:0;037c58a19ddb:37267-shortCompactions-0 {}] compactions.Compactor(225): Compacting 0e15f225c9fc496c9feeb6c7cfeaf92b, keycount=100, bloomtype=ROW, size=9.4 K, encoding=NONE, compression=NONE, seqNum=128, earliestPutTs=1731751199398 2024-11-16T10:00:01,394 INFO [RS:0;037c58a19ddb:37267-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 697cd6a363e514bff0d4ec954fc4034b#B#compaction#398 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-16T10:00:01,395 DEBUG [RS:0;037c58a19ddb:37267-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/697cd6a363e514bff0d4ec954fc4034b/.tmp/B/edb64d88ad5945048c2fa5d8962ee677 is 50, key is test_row_0/B:col10/1731751199022/Put/seqid=0 2024-11-16T10:00:01,395 INFO [RS:0;037c58a19ddb:37267-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 697cd6a363e514bff0d4ec954fc4034b#A#compaction#397 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-16T10:00:01,395 DEBUG [RS:0;037c58a19ddb:37267-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/697cd6a363e514bff0d4ec954fc4034b/.tmp/A/601acfc22f994b6ea38aa89aebc0dcc2 is 50, key is test_row_0/A:col10/1731751199022/Put/seqid=0 2024-11-16T10:00:01,401 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37591 is added to blk_1073742295_1471 (size=12104) 2024-11-16T10:00:01,402 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37591 is added to blk_1073742296_1472 (size=12104) 2024-11-16T10:00:01,450 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=37267 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=124 2024-11-16T10:00:01,450 DEBUG [RS_FLUSH_OPERATIONS-regionserver/037c58a19ddb:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731751192181.697cd6a363e514bff0d4ec954fc4034b. 2024-11-16T10:00:01,451 INFO [RS_FLUSH_OPERATIONS-regionserver/037c58a19ddb:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HRegion(2902): Flushing 697cd6a363e514bff0d4ec954fc4034b 3/3 column families, dataSize=140.89 KB heapSize=369.89 KB 2024-11-16T10:00:01,455 DEBUG [RS_FLUSH_OPERATIONS-regionserver/037c58a19ddb:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/697cd6a363e514bff0d4ec954fc4034b/.tmp/A/47804894c124483281dc4ec91f40d7a4 is 50, key is test_row_0/A:col10/1731751200598/Put/seqid=0 2024-11-16T10:00:01,458 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37591 is added to blk_1073742297_1473 (size=12151) 2024-11-16T10:00:01,760 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] regionserver.HRegion(8855): Flush requested on 697cd6a363e514bff0d4ec954fc4034b 2024-11-16T10:00:01,760 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731751192181.697cd6a363e514bff0d4ec954fc4034b. as already flushing 2024-11-16T10:00:01,785 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=697cd6a363e514bff0d4ec954fc4034b, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T10:00:01,786 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36022 deadline: 1731751261776, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=697cd6a363e514bff0d4ec954fc4034b, server=037c58a19ddb,37267,1731751077844 2024-11-16T10:00:01,786 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=697cd6a363e514bff0d4ec954fc4034b, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T10:00:01,786 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731751192181.697cd6a363e514bff0d4ec954fc4034b., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 , the old value is region=TestAcidGuarantees,,1731751192181.697cd6a363e514bff0d4ec954fc4034b., hostname=037c58a19ddb,37267,1731751077844, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=697cd6a363e514bff0d4ec954fc4034b, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:00:01,786 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731751192181.697cd6a363e514bff0d4ec954fc4034b., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=697cd6a363e514bff0d4ec954fc4034b, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:00:01,786 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36012 deadline: 1731751261777, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=697cd6a363e514bff0d4ec954fc4034b, server=037c58a19ddb,37267,1731751077844 2024-11-16T10:00:01,786 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731751192181.697cd6a363e514bff0d4ec954fc4034b., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 because the exception is null or not the one we care about 2024-11-16T10:00:01,786 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=697cd6a363e514bff0d4ec954fc4034b, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T10:00:01,786 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36058 deadline: 1731751261779, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=697cd6a363e514bff0d4ec954fc4034b, server=037c58a19ddb,37267,1731751077844 2024-11-16T10:00:01,786 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731751192181.697cd6a363e514bff0d4ec954fc4034b., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 , the old value is region=TestAcidGuarantees,,1731751192181.697cd6a363e514bff0d4ec954fc4034b., hostname=037c58a19ddb,37267,1731751077844, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=697cd6a363e514bff0d4ec954fc4034b, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:00:01,786 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731751192181.697cd6a363e514bff0d4ec954fc4034b., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=697cd6a363e514bff0d4ec954fc4034b, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:00:01,786 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731751192181.697cd6a363e514bff0d4ec954fc4034b., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 because the exception is null or not the one we care about 2024-11-16T10:00:01,786 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731751192181.697cd6a363e514bff0d4ec954fc4034b., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 , the old value is region=TestAcidGuarantees,,1731751192181.697cd6a363e514bff0d4ec954fc4034b., hostname=037c58a19ddb,37267,1731751077844, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=697cd6a363e514bff0d4ec954fc4034b, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:00:01,786 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731751192181.697cd6a363e514bff0d4ec954fc4034b., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=697cd6a363e514bff0d4ec954fc4034b, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:00:01,786 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731751192181.697cd6a363e514bff0d4ec954fc4034b., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 because the exception is null or not the one we care about 2024-11-16T10:00:01,807 DEBUG [RS:0;037c58a19ddb:37267-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/697cd6a363e514bff0d4ec954fc4034b/.tmp/A/601acfc22f994b6ea38aa89aebc0dcc2 as hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/697cd6a363e514bff0d4ec954fc4034b/A/601acfc22f994b6ea38aa89aebc0dcc2 2024-11-16T10:00:01,807 DEBUG [RS:0;037c58a19ddb:37267-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/697cd6a363e514bff0d4ec954fc4034b/.tmp/B/edb64d88ad5945048c2fa5d8962ee677 as hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/697cd6a363e514bff0d4ec954fc4034b/B/edb64d88ad5945048c2fa5d8962ee677 2024-11-16T10:00:01,814 INFO [RS:0;037c58a19ddb:37267-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 697cd6a363e514bff0d4ec954fc4034b/B of 697cd6a363e514bff0d4ec954fc4034b into edb64d88ad5945048c2fa5d8962ee677(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-16T10:00:01,814 INFO [RS:0;037c58a19ddb:37267-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 697cd6a363e514bff0d4ec954fc4034b/A of 697cd6a363e514bff0d4ec954fc4034b into 601acfc22f994b6ea38aa89aebc0dcc2(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-16T10:00:01,814 DEBUG [RS:0;037c58a19ddb:37267-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 697cd6a363e514bff0d4ec954fc4034b: 2024-11-16T10:00:01,814 DEBUG [RS:0;037c58a19ddb:37267-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 697cd6a363e514bff0d4ec954fc4034b: 2024-11-16T10:00:01,814 INFO [RS:0;037c58a19ddb:37267-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731751192181.697cd6a363e514bff0d4ec954fc4034b., storeName=697cd6a363e514bff0d4ec954fc4034b/B, priority=13, startTime=1731751201385; duration=0sec 2024-11-16T10:00:01,814 INFO [RS:0;037c58a19ddb:37267-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731751192181.697cd6a363e514bff0d4ec954fc4034b., storeName=697cd6a363e514bff0d4ec954fc4034b/A, priority=13, startTime=1731751201385; duration=0sec 2024-11-16T10:00:01,814 DEBUG [RS:0;037c58a19ddb:37267-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-16T10:00:01,814 DEBUG [RS:0;037c58a19ddb:37267-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-16T10:00:01,814 DEBUG [RS:0;037c58a19ddb:37267-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 697cd6a363e514bff0d4ec954fc4034b:B 2024-11-16T10:00:01,814 DEBUG [RS:0;037c58a19ddb:37267-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 697cd6a363e514bff0d4ec954fc4034b:A 2024-11-16T10:00:01,814 DEBUG [RS:0;037c58a19ddb:37267-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-16T10:00:01,815 DEBUG [RS:0;037c58a19ddb:37267-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 33865 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-16T10:00:01,815 DEBUG [RS:0;037c58a19ddb:37267-longCompactions-0 {}] regionserver.HStore(1541): 697cd6a363e514bff0d4ec954fc4034b/C is initiating minor compaction (all files) 2024-11-16T10:00:01,815 INFO [RS:0;037c58a19ddb:37267-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 697cd6a363e514bff0d4ec954fc4034b/C in TestAcidGuarantees,,1731751192181.697cd6a363e514bff0d4ec954fc4034b. 2024-11-16T10:00:01,815 INFO [RS:0;037c58a19ddb:37267-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/697cd6a363e514bff0d4ec954fc4034b/C/1ca86ee061304d5baf51f01fb8c60b7b, hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/697cd6a363e514bff0d4ec954fc4034b/C/e2fa6223e0924717854b53c074fd39fd, hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/697cd6a363e514bff0d4ec954fc4034b/C/9b994a5406404b3fa89fea7191ed65e7] into tmpdir=hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/697cd6a363e514bff0d4ec954fc4034b/.tmp, totalSize=33.1 K 2024-11-16T10:00:01,816 DEBUG [RS:0;037c58a19ddb:37267-longCompactions-0 {}] compactions.Compactor(225): Compacting 1ca86ee061304d5baf51f01fb8c60b7b, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=88, earliestPutTs=1731751198888 2024-11-16T10:00:01,816 DEBUG [RS:0;037c58a19ddb:37267-longCompactions-0 {}] compactions.Compactor(225): Compacting e2fa6223e0924717854b53c074fd39fd, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=113, earliestPutTs=1731751199022 2024-11-16T10:00:01,816 DEBUG [RS:0;037c58a19ddb:37267-longCompactions-0 {}] compactions.Compactor(225): Compacting 9b994a5406404b3fa89fea7191ed65e7, keycount=100, bloomtype=ROW, size=9.4 K, encoding=NONE, compression=NONE, seqNum=128, earliestPutTs=1731751199398 2024-11-16T10:00:01,822 INFO [RS:0;037c58a19ddb:37267-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 697cd6a363e514bff0d4ec954fc4034b#C#compaction#400 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-16T10:00:01,822 DEBUG [RS:0;037c58a19ddb:37267-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/697cd6a363e514bff0d4ec954fc4034b/.tmp/C/336e6611f01c42eca208c40d0f7121de is 50, key is test_row_0/C:col10/1731751199022/Put/seqid=0 2024-11-16T10:00:01,823 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37877 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=123 2024-11-16T10:00:01,834 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37591 is added to blk_1073742298_1474 (size=12104) 2024-11-16T10:00:01,859 INFO [RS_FLUSH_OPERATIONS-regionserver/037c58a19ddb:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=152 (bloomFilter=true), to=hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/697cd6a363e514bff0d4ec954fc4034b/.tmp/A/47804894c124483281dc4ec91f40d7a4 2024-11-16T10:00:01,866 DEBUG [RS_FLUSH_OPERATIONS-regionserver/037c58a19ddb:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/697cd6a363e514bff0d4ec954fc4034b/.tmp/B/1d367eb7c0064329aaffa8b4aaa027d7 is 50, key is test_row_0/B:col10/1731751200598/Put/seqid=0 2024-11-16T10:00:01,870 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37591 is added to blk_1073742299_1475 (size=12151) 2024-11-16T10:00:01,903 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=697cd6a363e514bff0d4ec954fc4034b, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T10:00:01,903 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36022 deadline: 1731751261893, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=697cd6a363e514bff0d4ec954fc4034b, server=037c58a19ddb,37267,1731751077844 2024-11-16T10:00:01,903 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=697cd6a363e514bff0d4ec954fc4034b, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T10:00:01,903 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731751192181.697cd6a363e514bff0d4ec954fc4034b., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 , the old value is region=TestAcidGuarantees,,1731751192181.697cd6a363e514bff0d4ec954fc4034b., hostname=037c58a19ddb,37267,1731751077844, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=697cd6a363e514bff0d4ec954fc4034b, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:00:01,903 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731751192181.697cd6a363e514bff0d4ec954fc4034b., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=697cd6a363e514bff0d4ec954fc4034b, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:00:01,903 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36012 deadline: 1731751261893, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=697cd6a363e514bff0d4ec954fc4034b, server=037c58a19ddb,37267,1731751077844 2024-11-16T10:00:01,903 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731751192181.697cd6a363e514bff0d4ec954fc4034b., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 because the exception is null or not the one we care about 2024-11-16T10:00:01,903 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=697cd6a363e514bff0d4ec954fc4034b, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T10:00:01,903 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36058 deadline: 1731751261893, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=697cd6a363e514bff0d4ec954fc4034b, server=037c58a19ddb,37267,1731751077844 2024-11-16T10:00:01,903 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731751192181.697cd6a363e514bff0d4ec954fc4034b., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 , the old value is region=TestAcidGuarantees,,1731751192181.697cd6a363e514bff0d4ec954fc4034b., hostname=037c58a19ddb,37267,1731751077844, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=697cd6a363e514bff0d4ec954fc4034b, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:00:01,903 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731751192181.697cd6a363e514bff0d4ec954fc4034b., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=697cd6a363e514bff0d4ec954fc4034b, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:00:01,903 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731751192181.697cd6a363e514bff0d4ec954fc4034b., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 because the exception is null or not the one we care about 2024-11-16T10:00:01,904 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731751192181.697cd6a363e514bff0d4ec954fc4034b., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 , the old value is region=TestAcidGuarantees,,1731751192181.697cd6a363e514bff0d4ec954fc4034b., hostname=037c58a19ddb,37267,1731751077844, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=697cd6a363e514bff0d4ec954fc4034b, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:00:01,904 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731751192181.697cd6a363e514bff0d4ec954fc4034b., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=697cd6a363e514bff0d4ec954fc4034b, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:00:01,904 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731751192181.697cd6a363e514bff0d4ec954fc4034b., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 because the exception is null or not the one we care about 2024-11-16T10:00:02,121 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=697cd6a363e514bff0d4ec954fc4034b, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T10:00:02,121 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36022 deadline: 1731751262113, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=697cd6a363e514bff0d4ec954fc4034b, server=037c58a19ddb,37267,1731751077844 2024-11-16T10:00:02,121 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=697cd6a363e514bff0d4ec954fc4034b, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T10:00:02,121 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36012 deadline: 1731751262113, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=697cd6a363e514bff0d4ec954fc4034b, server=037c58a19ddb,37267,1731751077844 2024-11-16T10:00:02,121 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731751192181.697cd6a363e514bff0d4ec954fc4034b., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 , the old value is region=TestAcidGuarantees,,1731751192181.697cd6a363e514bff0d4ec954fc4034b., hostname=037c58a19ddb,37267,1731751077844, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=697cd6a363e514bff0d4ec954fc4034b, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:00:02,121 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731751192181.697cd6a363e514bff0d4ec954fc4034b., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=697cd6a363e514bff0d4ec954fc4034b, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:00:02,121 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731751192181.697cd6a363e514bff0d4ec954fc4034b., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 because the exception is null or not the one we care about 2024-11-16T10:00:02,121 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=697cd6a363e514bff0d4ec954fc4034b, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T10:00:02,121 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36058 deadline: 1731751262114, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=697cd6a363e514bff0d4ec954fc4034b, server=037c58a19ddb,37267,1731751077844 2024-11-16T10:00:02,122 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731751192181.697cd6a363e514bff0d4ec954fc4034b., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 , the old value is region=TestAcidGuarantees,,1731751192181.697cd6a363e514bff0d4ec954fc4034b., hostname=037c58a19ddb,37267,1731751077844, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=697cd6a363e514bff0d4ec954fc4034b, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:00:02,122 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731751192181.697cd6a363e514bff0d4ec954fc4034b., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=697cd6a363e514bff0d4ec954fc4034b, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:00:02,122 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731751192181.697cd6a363e514bff0d4ec954fc4034b., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 because the exception is null or not the one we care about 2024-11-16T10:00:02,122 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731751192181.697cd6a363e514bff0d4ec954fc4034b., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 , the old value is region=TestAcidGuarantees,,1731751192181.697cd6a363e514bff0d4ec954fc4034b., hostname=037c58a19ddb,37267,1731751077844, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=697cd6a363e514bff0d4ec954fc4034b, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:00:02,122 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731751192181.697cd6a363e514bff0d4ec954fc4034b., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=697cd6a363e514bff0d4ec954fc4034b, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:00:02,122 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731751192181.697cd6a363e514bff0d4ec954fc4034b., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 because the exception is null or not the one we care about 2024-11-16T10:00:02,238 DEBUG [RS:0;037c58a19ddb:37267-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/697cd6a363e514bff0d4ec954fc4034b/.tmp/C/336e6611f01c42eca208c40d0f7121de as hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/697cd6a363e514bff0d4ec954fc4034b/C/336e6611f01c42eca208c40d0f7121de 2024-11-16T10:00:02,243 INFO [RS:0;037c58a19ddb:37267-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 697cd6a363e514bff0d4ec954fc4034b/C of 697cd6a363e514bff0d4ec954fc4034b into 336e6611f01c42eca208c40d0f7121de(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-16T10:00:02,243 DEBUG [RS:0;037c58a19ddb:37267-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 697cd6a363e514bff0d4ec954fc4034b: 2024-11-16T10:00:02,243 INFO [RS:0;037c58a19ddb:37267-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731751192181.697cd6a363e514bff0d4ec954fc4034b., storeName=697cd6a363e514bff0d4ec954fc4034b/C, priority=13, startTime=1731751201385; duration=0sec 2024-11-16T10:00:02,243 DEBUG [RS:0;037c58a19ddb:37267-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-16T10:00:02,243 DEBUG [RS:0;037c58a19ddb:37267-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 697cd6a363e514bff0d4ec954fc4034b:C 2024-11-16T10:00:02,270 INFO [RS_FLUSH_OPERATIONS-regionserver/037c58a19ddb:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=152 (bloomFilter=true), to=hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/697cd6a363e514bff0d4ec954fc4034b/.tmp/B/1d367eb7c0064329aaffa8b4aaa027d7 2024-11-16T10:00:02,278 DEBUG [RS_FLUSH_OPERATIONS-regionserver/037c58a19ddb:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/697cd6a363e514bff0d4ec954fc4034b/.tmp/C/e4b523607a6e49e8a22c3232d2fa54ec is 50, key is test_row_0/C:col10/1731751200598/Put/seqid=0 2024-11-16T10:00:02,282 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37591 is added to blk_1073742300_1476 (size=12151) 2024-11-16T10:00:02,434 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=697cd6a363e514bff0d4ec954fc4034b, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T10:00:02,434 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36022 deadline: 1731751262433, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=697cd6a363e514bff0d4ec954fc4034b, server=037c58a19ddb,37267,1731751077844 2024-11-16T10:00:02,435 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=697cd6a363e514bff0d4ec954fc4034b, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T10:00:02,435 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731751192181.697cd6a363e514bff0d4ec954fc4034b., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 , the old value is region=TestAcidGuarantees,,1731751192181.697cd6a363e514bff0d4ec954fc4034b., hostname=037c58a19ddb,37267,1731751077844, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=697cd6a363e514bff0d4ec954fc4034b, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:00:02,435 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36012 deadline: 1731751262433, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=697cd6a363e514bff0d4ec954fc4034b, server=037c58a19ddb,37267,1731751077844 2024-11-16T10:00:02,435 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731751192181.697cd6a363e514bff0d4ec954fc4034b., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=697cd6a363e514bff0d4ec954fc4034b, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:00:02,435 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731751192181.697cd6a363e514bff0d4ec954fc4034b., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 because the exception is null or not the one we care about 2024-11-16T10:00:02,435 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=697cd6a363e514bff0d4ec954fc4034b, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T10:00:02,435 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731751192181.697cd6a363e514bff0d4ec954fc4034b., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 , the old value is region=TestAcidGuarantees,,1731751192181.697cd6a363e514bff0d4ec954fc4034b., hostname=037c58a19ddb,37267,1731751077844, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=697cd6a363e514bff0d4ec954fc4034b, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:00:02,435 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731751192181.697cd6a363e514bff0d4ec954fc4034b., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=697cd6a363e514bff0d4ec954fc4034b, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:00:02,435 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36058 deadline: 1731751262433, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=697cd6a363e514bff0d4ec954fc4034b, server=037c58a19ddb,37267,1731751077844 2024-11-16T10:00:02,435 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731751192181.697cd6a363e514bff0d4ec954fc4034b., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 because the exception is null or not the one we care about 2024-11-16T10:00:02,435 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731751192181.697cd6a363e514bff0d4ec954fc4034b., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 , the old value is region=TestAcidGuarantees,,1731751192181.697cd6a363e514bff0d4ec954fc4034b., hostname=037c58a19ddb,37267,1731751077844, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=697cd6a363e514bff0d4ec954fc4034b, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:00:02,435 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731751192181.697cd6a363e514bff0d4ec954fc4034b., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=697cd6a363e514bff0d4ec954fc4034b, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:00:02,435 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731751192181.697cd6a363e514bff0d4ec954fc4034b., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 because the exception is null or not the one we care about 2024-11-16T10:00:02,683 INFO [RS_FLUSH_OPERATIONS-regionserver/037c58a19ddb:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=152 (bloomFilter=true), to=hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/697cd6a363e514bff0d4ec954fc4034b/.tmp/C/e4b523607a6e49e8a22c3232d2fa54ec 2024-11-16T10:00:02,687 DEBUG [RS_FLUSH_OPERATIONS-regionserver/037c58a19ddb:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/697cd6a363e514bff0d4ec954fc4034b/.tmp/A/47804894c124483281dc4ec91f40d7a4 as hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/697cd6a363e514bff0d4ec954fc4034b/A/47804894c124483281dc4ec91f40d7a4 2024-11-16T10:00:02,691 INFO [RS_FLUSH_OPERATIONS-regionserver/037c58a19ddb:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/697cd6a363e514bff0d4ec954fc4034b/A/47804894c124483281dc4ec91f40d7a4, entries=150, sequenceid=152, filesize=11.9 K 2024-11-16T10:00:02,691 DEBUG [RS_FLUSH_OPERATIONS-regionserver/037c58a19ddb:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/697cd6a363e514bff0d4ec954fc4034b/.tmp/B/1d367eb7c0064329aaffa8b4aaa027d7 as hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/697cd6a363e514bff0d4ec954fc4034b/B/1d367eb7c0064329aaffa8b4aaa027d7 2024-11-16T10:00:02,695 INFO [RS_FLUSH_OPERATIONS-regionserver/037c58a19ddb:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/697cd6a363e514bff0d4ec954fc4034b/B/1d367eb7c0064329aaffa8b4aaa027d7, entries=150, sequenceid=152, filesize=11.9 K 2024-11-16T10:00:02,695 DEBUG [RS_FLUSH_OPERATIONS-regionserver/037c58a19ddb:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/697cd6a363e514bff0d4ec954fc4034b/.tmp/C/e4b523607a6e49e8a22c3232d2fa54ec as hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/697cd6a363e514bff0d4ec954fc4034b/C/e4b523607a6e49e8a22c3232d2fa54ec 2024-11-16T10:00:02,699 INFO [RS_FLUSH_OPERATIONS-regionserver/037c58a19ddb:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/697cd6a363e514bff0d4ec954fc4034b/C/e4b523607a6e49e8a22c3232d2fa54ec, entries=150, sequenceid=152, filesize=11.9 K 2024-11-16T10:00:02,700 INFO [RS_FLUSH_OPERATIONS-regionserver/037c58a19ddb:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HRegion(3140): Finished flush of dataSize ~140.89 KB/144270, heapSize ~369.84 KB/378720, currentSize=60.38 KB/61830 for 697cd6a363e514bff0d4ec954fc4034b in 1249ms, sequenceid=152, compaction requested=false 2024-11-16T10:00:02,700 DEBUG [RS_FLUSH_OPERATIONS-regionserver/037c58a19ddb:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HRegion(2603): Flush status journal for 697cd6a363e514bff0d4ec954fc4034b: 2024-11-16T10:00:02,700 DEBUG [RS_FLUSH_OPERATIONS-regionserver/037c58a19ddb:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731751192181.697cd6a363e514bff0d4ec954fc4034b. 2024-11-16T10:00:02,700 DEBUG [RS_FLUSH_OPERATIONS-regionserver/037c58a19ddb:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=124 2024-11-16T10:00:02,700 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37877 {}] master.HMaster(4169): Remote procedure done, pid=124 2024-11-16T10:00:02,703 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=124, resume processing ppid=123 2024-11-16T10:00:02,703 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=124, ppid=123, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.0160 sec 2024-11-16T10:00:02,705 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=123, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=123, table=TestAcidGuarantees in 2.0200 sec 2024-11-16T10:00:02,833 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37877 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=123 2024-11-16T10:00:02,833 INFO [RPCClient-NioEventLoopGroup-4-10 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-16T10:00:02,834 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37877 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-16T10:00:02,835 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37877 {}] procedure2.ProcedureExecutor(1139): Stored pid=125, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=125, table=TestAcidGuarantees 2024-11-16T10:00:02,836 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=125, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=125, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-16T10:00:02,836 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37877 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=125 2024-11-16T10:00:02,837 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=125, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=125, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-16T10:00:02,837 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=126, ppid=125, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-16T10:00:02,943 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37877 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=125 2024-11-16T10:00:02,946 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] regionserver.HRegion(8855): Flush requested on 697cd6a363e514bff0d4ec954fc4034b 2024-11-16T10:00:02,946 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-16T10:00:02,946 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 697cd6a363e514bff0d4ec954fc4034b 3/3 column families, dataSize=67.09 KB heapSize=176.53 KB 2024-11-16T10:00:02,951 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/697cd6a363e514bff0d4ec954fc4034b/.tmp/A/75216ae3e27a4a6fa6d1f190faf54871 is 50, key is test_row_0/A:col10/1731751201775/Put/seqid=0 2024-11-16T10:00:02,959 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37591 is added to blk_1073742301_1477 (size=14541) 2024-11-16T10:00:02,964 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=168 (bloomFilter=true), to=hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/697cd6a363e514bff0d4ec954fc4034b/.tmp/A/75216ae3e27a4a6fa6d1f190faf54871 2024-11-16T10:00:02,971 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/697cd6a363e514bff0d4ec954fc4034b/.tmp/B/127fba5983df4cef875b7429842baf4e is 50, key is test_row_0/B:col10/1731751201775/Put/seqid=0 2024-11-16T10:00:02,975 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37591 is added to blk_1073742302_1478 (size=12151) 2024-11-16T10:00:02,976 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=168 (bloomFilter=true), to=hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/697cd6a363e514bff0d4ec954fc4034b/.tmp/B/127fba5983df4cef875b7429842baf4e 2024-11-16T10:00:02,988 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/697cd6a363e514bff0d4ec954fc4034b/.tmp/C/fd7057e35b004c38881704f7e925903e is 50, key is test_row_0/C:col10/1731751201775/Put/seqid=0 2024-11-16T10:00:02,990 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=37267 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=126 2024-11-16T10:00:02,991 DEBUG [RS_FLUSH_OPERATIONS-regionserver/037c58a19ddb:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731751192181.697cd6a363e514bff0d4ec954fc4034b. 2024-11-16T10:00:02,991 DEBUG [RS_FLUSH_OPERATIONS-regionserver/037c58a19ddb:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731751192181.697cd6a363e514bff0d4ec954fc4034b. as already flushing 2024-11-16T10:00:02,991 DEBUG [RS_FLUSH_OPERATIONS-regionserver/037c58a19ddb:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731751192181.697cd6a363e514bff0d4ec954fc4034b. 2024-11-16T10:00:02,991 ERROR [RS_FLUSH_OPERATIONS-regionserver/037c58a19ddb:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] handler.RSProcedureHandler(58): pid=126 java.io.IOException: Unable to complete flush {ENCODED => 697cd6a363e514bff0d4ec954fc4034b, NAME => 'TestAcidGuarantees,,1731751192181.697cd6a363e514bff0d4ec954fc4034b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T10:00:02,991 DEBUG [RS_FLUSH_OPERATIONS-regionserver/037c58a19ddb:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=126 java.io.IOException: Unable to complete flush {ENCODED => 697cd6a363e514bff0d4ec954fc4034b, NAME => 'TestAcidGuarantees,,1731751192181.697cd6a363e514bff0d4ec954fc4034b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T10:00:02,992 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37877 {}] master.HMaster(4177): Remote procedure failed, pid=126 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 697cd6a363e514bff0d4ec954fc4034b, NAME => 'TestAcidGuarantees,,1731751192181.697cd6a363e514bff0d4ec954fc4034b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 697cd6a363e514bff0d4ec954fc4034b, NAME => 'TestAcidGuarantees,,1731751192181.697cd6a363e514bff0d4ec954fc4034b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T10:00:02,996 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37591 is added to blk_1073742303_1479 (size=12151) 2024-11-16T10:00:03,027 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=697cd6a363e514bff0d4ec954fc4034b, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T10:00:03,027 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] ipc.CallRunner(138): callId: 74 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36058 deadline: 1731751263014, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=697cd6a363e514bff0d4ec954fc4034b, server=037c58a19ddb,37267,1731751077844 2024-11-16T10:00:03,027 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731751192181.697cd6a363e514bff0d4ec954fc4034b., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 , the old value is region=TestAcidGuarantees,,1731751192181.697cd6a363e514bff0d4ec954fc4034b., hostname=037c58a19ddb,37267,1731751077844, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=697cd6a363e514bff0d4ec954fc4034b, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:00:03,027 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731751192181.697cd6a363e514bff0d4ec954fc4034b., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=697cd6a363e514bff0d4ec954fc4034b, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:00:03,027 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731751192181.697cd6a363e514bff0d4ec954fc4034b., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 because the exception is null or not the one we care about 2024-11-16T10:00:03,027 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=697cd6a363e514bff0d4ec954fc4034b, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T10:00:03,027 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36022 deadline: 1731751263018, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=697cd6a363e514bff0d4ec954fc4034b, server=037c58a19ddb,37267,1731751077844 2024-11-16T10:00:03,028 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731751192181.697cd6a363e514bff0d4ec954fc4034b., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 , the old value is region=TestAcidGuarantees,,1731751192181.697cd6a363e514bff0d4ec954fc4034b., hostname=037c58a19ddb,37267,1731751077844, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=697cd6a363e514bff0d4ec954fc4034b, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:00:03,028 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731751192181.697cd6a363e514bff0d4ec954fc4034b., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=697cd6a363e514bff0d4ec954fc4034b, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:00:03,028 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731751192181.697cd6a363e514bff0d4ec954fc4034b., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 because the exception is null or not the one we care about 2024-11-16T10:00:03,035 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=697cd6a363e514bff0d4ec954fc4034b, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T10:00:03,035 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36012 deadline: 1731751263025, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=697cd6a363e514bff0d4ec954fc4034b, server=037c58a19ddb,37267,1731751077844 2024-11-16T10:00:03,036 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731751192181.697cd6a363e514bff0d4ec954fc4034b., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 , the old value is region=TestAcidGuarantees,,1731751192181.697cd6a363e514bff0d4ec954fc4034b., hostname=037c58a19ddb,37267,1731751077844, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=697cd6a363e514bff0d4ec954fc4034b, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:00:03,036 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731751192181.697cd6a363e514bff0d4ec954fc4034b., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=697cd6a363e514bff0d4ec954fc4034b, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:00:03,036 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731751192181.697cd6a363e514bff0d4ec954fc4034b., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 because the exception is null or not the one we care about 2024-11-16T10:00:03,142 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=697cd6a363e514bff0d4ec954fc4034b, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T10:00:03,142 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] ipc.CallRunner(138): callId: 75 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36058 deadline: 1731751263133, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=697cd6a363e514bff0d4ec954fc4034b, server=037c58a19ddb,37267,1731751077844 2024-11-16T10:00:03,142 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=697cd6a363e514bff0d4ec954fc4034b, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T10:00:03,142 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36022 deadline: 1731751263133, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=697cd6a363e514bff0d4ec954fc4034b, server=037c58a19ddb,37267,1731751077844 2024-11-16T10:00:03,142 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731751192181.697cd6a363e514bff0d4ec954fc4034b., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 , the old value is region=TestAcidGuarantees,,1731751192181.697cd6a363e514bff0d4ec954fc4034b., hostname=037c58a19ddb,37267,1731751077844, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=697cd6a363e514bff0d4ec954fc4034b, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:00:03,143 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731751192181.697cd6a363e514bff0d4ec954fc4034b., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=697cd6a363e514bff0d4ec954fc4034b, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:00:03,143 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731751192181.697cd6a363e514bff0d4ec954fc4034b., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 because the exception is null or not the one we care about 2024-11-16T10:00:03,143 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731751192181.697cd6a363e514bff0d4ec954fc4034b., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 , the old value is region=TestAcidGuarantees,,1731751192181.697cd6a363e514bff0d4ec954fc4034b., hostname=037c58a19ddb,37267,1731751077844, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=697cd6a363e514bff0d4ec954fc4034b, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:00:03,143 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731751192181.697cd6a363e514bff0d4ec954fc4034b., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=697cd6a363e514bff0d4ec954fc4034b, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:00:03,143 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731751192181.697cd6a363e514bff0d4ec954fc4034b., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 because the exception is null or not the one we care about 2024-11-16T10:00:03,144 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=37267 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=126 2024-11-16T10:00:03,144 DEBUG [RS_FLUSH_OPERATIONS-regionserver/037c58a19ddb:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731751192181.697cd6a363e514bff0d4ec954fc4034b. 2024-11-16T10:00:03,144 DEBUG [RS_FLUSH_OPERATIONS-regionserver/037c58a19ddb:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731751192181.697cd6a363e514bff0d4ec954fc4034b. as already flushing 2024-11-16T10:00:03,144 DEBUG [RS_FLUSH_OPERATIONS-regionserver/037c58a19ddb:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731751192181.697cd6a363e514bff0d4ec954fc4034b. 2024-11-16T10:00:03,145 ERROR [RS_FLUSH_OPERATIONS-regionserver/037c58a19ddb:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] handler.RSProcedureHandler(58): pid=126 java.io.IOException: Unable to complete flush {ENCODED => 697cd6a363e514bff0d4ec954fc4034b, NAME => 'TestAcidGuarantees,,1731751192181.697cd6a363e514bff0d4ec954fc4034b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T10:00:03,145 DEBUG [RS_FLUSH_OPERATIONS-regionserver/037c58a19ddb:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=126 java.io.IOException: Unable to complete flush {ENCODED => 697cd6a363e514bff0d4ec954fc4034b, NAME => 'TestAcidGuarantees,,1731751192181.697cd6a363e514bff0d4ec954fc4034b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T10:00:03,145 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37877 {}] master.HMaster(4177): Remote procedure failed, pid=126 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 697cd6a363e514bff0d4ec954fc4034b, NAME => 'TestAcidGuarantees,,1731751192181.697cd6a363e514bff0d4ec954fc4034b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 697cd6a363e514bff0d4ec954fc4034b, NAME => 'TestAcidGuarantees,,1731751192181.697cd6a363e514bff0d4ec954fc4034b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T10:00:03,151 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=697cd6a363e514bff0d4ec954fc4034b, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T10:00:03,151 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36012 deadline: 1731751263142, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=697cd6a363e514bff0d4ec954fc4034b, server=037c58a19ddb,37267,1731751077844 2024-11-16T10:00:03,151 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731751192181.697cd6a363e514bff0d4ec954fc4034b., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 , the old value is region=TestAcidGuarantees,,1731751192181.697cd6a363e514bff0d4ec954fc4034b., hostname=037c58a19ddb,37267,1731751077844, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=697cd6a363e514bff0d4ec954fc4034b, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:00:03,151 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731751192181.697cd6a363e514bff0d4ec954fc4034b., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=697cd6a363e514bff0d4ec954fc4034b, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:00:03,151 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731751192181.697cd6a363e514bff0d4ec954fc4034b., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 because the exception is null or not the one we care about 2024-11-16T10:00:03,153 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37877 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=125 2024-11-16T10:00:03,297 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=37267 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=126 2024-11-16T10:00:03,297 DEBUG [RS_FLUSH_OPERATIONS-regionserver/037c58a19ddb:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731751192181.697cd6a363e514bff0d4ec954fc4034b. 2024-11-16T10:00:03,298 DEBUG [RS_FLUSH_OPERATIONS-regionserver/037c58a19ddb:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731751192181.697cd6a363e514bff0d4ec954fc4034b. as already flushing 2024-11-16T10:00:03,298 DEBUG [RS_FLUSH_OPERATIONS-regionserver/037c58a19ddb:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731751192181.697cd6a363e514bff0d4ec954fc4034b. 2024-11-16T10:00:03,298 ERROR [RS_FLUSH_OPERATIONS-regionserver/037c58a19ddb:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] handler.RSProcedureHandler(58): pid=126 java.io.IOException: Unable to complete flush {ENCODED => 697cd6a363e514bff0d4ec954fc4034b, NAME => 'TestAcidGuarantees,,1731751192181.697cd6a363e514bff0d4ec954fc4034b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T10:00:03,298 DEBUG [RS_FLUSH_OPERATIONS-regionserver/037c58a19ddb:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=126 java.io.IOException: Unable to complete flush {ENCODED => 697cd6a363e514bff0d4ec954fc4034b, NAME => 'TestAcidGuarantees,,1731751192181.697cd6a363e514bff0d4ec954fc4034b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T10:00:03,298 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37877 {}] master.HMaster(4177): Remote procedure failed, pid=126 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 697cd6a363e514bff0d4ec954fc4034b, NAME => 'TestAcidGuarantees,,1731751192181.697cd6a363e514bff0d4ec954fc4034b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 697cd6a363e514bff0d4ec954fc4034b, NAME => 'TestAcidGuarantees,,1731751192181.697cd6a363e514bff0d4ec954fc4034b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T10:00:03,357 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=697cd6a363e514bff0d4ec954fc4034b, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T10:00:03,357 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] ipc.CallRunner(138): callId: 76 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36058 deadline: 1731751263353, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=697cd6a363e514bff0d4ec954fc4034b, server=037c58a19ddb,37267,1731751077844 2024-11-16T10:00:03,358 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=697cd6a363e514bff0d4ec954fc4034b, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T10:00:03,358 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731751192181.697cd6a363e514bff0d4ec954fc4034b., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 , the old value is region=TestAcidGuarantees,,1731751192181.697cd6a363e514bff0d4ec954fc4034b., hostname=037c58a19ddb,37267,1731751077844, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=697cd6a363e514bff0d4ec954fc4034b, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:00:03,358 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731751192181.697cd6a363e514bff0d4ec954fc4034b., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=697cd6a363e514bff0d4ec954fc4034b, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:00:03,358 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36022 deadline: 1731751263353, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=697cd6a363e514bff0d4ec954fc4034b, server=037c58a19ddb,37267,1731751077844 2024-11-16T10:00:03,358 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731751192181.697cd6a363e514bff0d4ec954fc4034b., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 because the exception is null or not the one we care about 2024-11-16T10:00:03,358 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731751192181.697cd6a363e514bff0d4ec954fc4034b., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 , the old value is region=TestAcidGuarantees,,1731751192181.697cd6a363e514bff0d4ec954fc4034b., hostname=037c58a19ddb,37267,1731751077844, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=697cd6a363e514bff0d4ec954fc4034b, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:00:03,358 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731751192181.697cd6a363e514bff0d4ec954fc4034b., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=697cd6a363e514bff0d4ec954fc4034b, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:00:03,358 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731751192181.697cd6a363e514bff0d4ec954fc4034b., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 because the exception is null or not the one we care about 2024-11-16T10:00:03,368 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=697cd6a363e514bff0d4ec954fc4034b, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T10:00:03,369 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36012 deadline: 1731751263363, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=697cd6a363e514bff0d4ec954fc4034b, server=037c58a19ddb,37267,1731751077844 2024-11-16T10:00:03,369 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731751192181.697cd6a363e514bff0d4ec954fc4034b., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 , the old value is region=TestAcidGuarantees,,1731751192181.697cd6a363e514bff0d4ec954fc4034b., hostname=037c58a19ddb,37267,1731751077844, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=697cd6a363e514bff0d4ec954fc4034b, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:00:03,369 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731751192181.697cd6a363e514bff0d4ec954fc4034b., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=697cd6a363e514bff0d4ec954fc4034b, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:00:03,369 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731751192181.697cd6a363e514bff0d4ec954fc4034b., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 because the exception is null or not the one we care about 2024-11-16T10:00:03,396 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=168 (bloomFilter=true), to=hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/697cd6a363e514bff0d4ec954fc4034b/.tmp/C/fd7057e35b004c38881704f7e925903e 2024-11-16T10:00:03,401 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/697cd6a363e514bff0d4ec954fc4034b/.tmp/A/75216ae3e27a4a6fa6d1f190faf54871 as hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/697cd6a363e514bff0d4ec954fc4034b/A/75216ae3e27a4a6fa6d1f190faf54871 2024-11-16T10:00:03,405 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/697cd6a363e514bff0d4ec954fc4034b/A/75216ae3e27a4a6fa6d1f190faf54871, entries=200, sequenceid=168, filesize=14.2 K 2024-11-16T10:00:03,405 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/697cd6a363e514bff0d4ec954fc4034b/.tmp/B/127fba5983df4cef875b7429842baf4e as hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/697cd6a363e514bff0d4ec954fc4034b/B/127fba5983df4cef875b7429842baf4e 2024-11-16T10:00:03,409 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/697cd6a363e514bff0d4ec954fc4034b/B/127fba5983df4cef875b7429842baf4e, entries=150, sequenceid=168, filesize=11.9 K 2024-11-16T10:00:03,410 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/697cd6a363e514bff0d4ec954fc4034b/.tmp/C/fd7057e35b004c38881704f7e925903e as hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/697cd6a363e514bff0d4ec954fc4034b/C/fd7057e35b004c38881704f7e925903e 2024-11-16T10:00:03,413 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/697cd6a363e514bff0d4ec954fc4034b/C/fd7057e35b004c38881704f7e925903e, entries=150, sequenceid=168, filesize=11.9 K 2024-11-16T10:00:03,414 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~67.09 KB/68700, heapSize ~176.48 KB/180720, currentSize=134.18 KB/137400 for 697cd6a363e514bff0d4ec954fc4034b in 468ms, sequenceid=168, compaction requested=true 2024-11-16T10:00:03,414 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 697cd6a363e514bff0d4ec954fc4034b: 2024-11-16T10:00:03,414 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 697cd6a363e514bff0d4ec954fc4034b:A, priority=-2147483648, current under compaction store size is 1 2024-11-16T10:00:03,414 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-16T10:00:03,414 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 697cd6a363e514bff0d4ec954fc4034b:B, priority=-2147483648, current under compaction store size is 2 2024-11-16T10:00:03,414 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-16T10:00:03,414 DEBUG [RS:0;037c58a19ddb:37267-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-16T10:00:03,414 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 697cd6a363e514bff0d4ec954fc4034b:C, priority=-2147483648, current under compaction store size is 3 2024-11-16T10:00:03,414 DEBUG [RS:0;037c58a19ddb:37267-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-16T10:00:03,414 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-16T10:00:03,416 DEBUG [RS:0;037c58a19ddb:37267-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36406 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-16T10:00:03,416 DEBUG [RS:0;037c58a19ddb:37267-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 38796 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-16T10:00:03,416 DEBUG [RS:0;037c58a19ddb:37267-shortCompactions-0 {}] regionserver.HStore(1541): 697cd6a363e514bff0d4ec954fc4034b/A is initiating minor compaction (all files) 2024-11-16T10:00:03,416 DEBUG [RS:0;037c58a19ddb:37267-longCompactions-0 {}] regionserver.HStore(1541): 697cd6a363e514bff0d4ec954fc4034b/B is initiating minor compaction (all files) 2024-11-16T10:00:03,416 INFO [RS:0;037c58a19ddb:37267-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 697cd6a363e514bff0d4ec954fc4034b/A in TestAcidGuarantees,,1731751192181.697cd6a363e514bff0d4ec954fc4034b. 2024-11-16T10:00:03,416 INFO [RS:0;037c58a19ddb:37267-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 697cd6a363e514bff0d4ec954fc4034b/B in TestAcidGuarantees,,1731751192181.697cd6a363e514bff0d4ec954fc4034b. 2024-11-16T10:00:03,416 INFO [RS:0;037c58a19ddb:37267-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/697cd6a363e514bff0d4ec954fc4034b/A/601acfc22f994b6ea38aa89aebc0dcc2, hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/697cd6a363e514bff0d4ec954fc4034b/A/47804894c124483281dc4ec91f40d7a4, hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/697cd6a363e514bff0d4ec954fc4034b/A/75216ae3e27a4a6fa6d1f190faf54871] into tmpdir=hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/697cd6a363e514bff0d4ec954fc4034b/.tmp, totalSize=37.9 K 2024-11-16T10:00:03,416 INFO [RS:0;037c58a19ddb:37267-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/697cd6a363e514bff0d4ec954fc4034b/B/edb64d88ad5945048c2fa5d8962ee677, hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/697cd6a363e514bff0d4ec954fc4034b/B/1d367eb7c0064329aaffa8b4aaa027d7, hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/697cd6a363e514bff0d4ec954fc4034b/B/127fba5983df4cef875b7429842baf4e] into tmpdir=hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/697cd6a363e514bff0d4ec954fc4034b/.tmp, totalSize=35.6 K 2024-11-16T10:00:03,416 DEBUG [RS:0;037c58a19ddb:37267-shortCompactions-0 {}] compactions.Compactor(225): Compacting 601acfc22f994b6ea38aa89aebc0dcc2, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=128, earliestPutTs=1731751199022 2024-11-16T10:00:03,416 DEBUG [RS:0;037c58a19ddb:37267-longCompactions-0 {}] compactions.Compactor(225): Compacting edb64d88ad5945048c2fa5d8962ee677, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=128, earliestPutTs=1731751199022 2024-11-16T10:00:03,417 DEBUG [RS:0;037c58a19ddb:37267-shortCompactions-0 {}] compactions.Compactor(225): Compacting 47804894c124483281dc4ec91f40d7a4, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=152, earliestPutTs=1731751200585 2024-11-16T10:00:03,417 DEBUG [RS:0;037c58a19ddb:37267-longCompactions-0 {}] compactions.Compactor(225): Compacting 1d367eb7c0064329aaffa8b4aaa027d7, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=152, earliestPutTs=1731751200585 2024-11-16T10:00:03,417 DEBUG [RS:0;037c58a19ddb:37267-shortCompactions-0 {}] compactions.Compactor(225): Compacting 75216ae3e27a4a6fa6d1f190faf54871, keycount=200, bloomtype=ROW, size=14.2 K, encoding=NONE, compression=NONE, seqNum=168, earliestPutTs=1731751201775 2024-11-16T10:00:03,417 DEBUG [RS:0;037c58a19ddb:37267-longCompactions-0 {}] compactions.Compactor(225): Compacting 127fba5983df4cef875b7429842baf4e, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=168, earliestPutTs=1731751201775 2024-11-16T10:00:03,425 INFO [RS:0;037c58a19ddb:37267-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 697cd6a363e514bff0d4ec954fc4034b#A#compaction#406 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-16T10:00:03,425 INFO [RS:0;037c58a19ddb:37267-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 697cd6a363e514bff0d4ec954fc4034b#B#compaction#407 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-16T10:00:03,425 DEBUG [RS:0;037c58a19ddb:37267-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/697cd6a363e514bff0d4ec954fc4034b/.tmp/A/387791a6a1904f44b3973f11b149c72a is 50, key is test_row_0/A:col10/1731751201775/Put/seqid=0 2024-11-16T10:00:03,425 DEBUG [RS:0;037c58a19ddb:37267-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/697cd6a363e514bff0d4ec954fc4034b/.tmp/B/173c5e49d87a4489abab8fc2e7bdade3 is 50, key is test_row_0/B:col10/1731751201775/Put/seqid=0 2024-11-16T10:00:03,431 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37591 is added to blk_1073742305_1481 (size=12357) 2024-11-16T10:00:03,432 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37591 is added to blk_1073742304_1480 (size=12357) 2024-11-16T10:00:03,438 DEBUG [RS:0;037c58a19ddb:37267-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/697cd6a363e514bff0d4ec954fc4034b/.tmp/A/387791a6a1904f44b3973f11b149c72a as hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/697cd6a363e514bff0d4ec954fc4034b/A/387791a6a1904f44b3973f11b149c72a 2024-11-16T10:00:03,443 INFO [RS:0;037c58a19ddb:37267-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 697cd6a363e514bff0d4ec954fc4034b/A of 697cd6a363e514bff0d4ec954fc4034b into 387791a6a1904f44b3973f11b149c72a(size=12.1 K), total size for store is 12.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-16T10:00:03,443 DEBUG [RS:0;037c58a19ddb:37267-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 697cd6a363e514bff0d4ec954fc4034b: 2024-11-16T10:00:03,443 INFO [RS:0;037c58a19ddb:37267-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731751192181.697cd6a363e514bff0d4ec954fc4034b., storeName=697cd6a363e514bff0d4ec954fc4034b/A, priority=13, startTime=1731751203414; duration=0sec 2024-11-16T10:00:03,443 DEBUG [RS:0;037c58a19ddb:37267-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-16T10:00:03,443 DEBUG [RS:0;037c58a19ddb:37267-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 697cd6a363e514bff0d4ec954fc4034b:A 2024-11-16T10:00:03,443 DEBUG [RS:0;037c58a19ddb:37267-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-16T10:00:03,444 DEBUG [RS:0;037c58a19ddb:37267-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36406 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-16T10:00:03,444 DEBUG [RS:0;037c58a19ddb:37267-shortCompactions-0 {}] regionserver.HStore(1541): 697cd6a363e514bff0d4ec954fc4034b/C is initiating minor compaction (all files) 2024-11-16T10:00:03,444 INFO [RS:0;037c58a19ddb:37267-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 697cd6a363e514bff0d4ec954fc4034b/C in TestAcidGuarantees,,1731751192181.697cd6a363e514bff0d4ec954fc4034b. 2024-11-16T10:00:03,444 INFO [RS:0;037c58a19ddb:37267-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/697cd6a363e514bff0d4ec954fc4034b/C/336e6611f01c42eca208c40d0f7121de, hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/697cd6a363e514bff0d4ec954fc4034b/C/e4b523607a6e49e8a22c3232d2fa54ec, hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/697cd6a363e514bff0d4ec954fc4034b/C/fd7057e35b004c38881704f7e925903e] into tmpdir=hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/697cd6a363e514bff0d4ec954fc4034b/.tmp, totalSize=35.6 K 2024-11-16T10:00:03,444 DEBUG [RS:0;037c58a19ddb:37267-shortCompactions-0 {}] compactions.Compactor(225): Compacting 336e6611f01c42eca208c40d0f7121de, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=128, earliestPutTs=1731751199022 2024-11-16T10:00:03,445 DEBUG [RS:0;037c58a19ddb:37267-shortCompactions-0 {}] compactions.Compactor(225): Compacting e4b523607a6e49e8a22c3232d2fa54ec, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=152, earliestPutTs=1731751200585 2024-11-16T10:00:03,445 DEBUG [RS:0;037c58a19ddb:37267-shortCompactions-0 {}] compactions.Compactor(225): Compacting fd7057e35b004c38881704f7e925903e, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=168, earliestPutTs=1731751201775 2024-11-16T10:00:03,450 INFO [RS:0;037c58a19ddb:37267-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 697cd6a363e514bff0d4ec954fc4034b#C#compaction#408 average throughput is unlimited, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-16T10:00:03,451 DEBUG [RS:0;037c58a19ddb:37267-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/697cd6a363e514bff0d4ec954fc4034b/.tmp/C/1eda3944ab0448908f50204d73526559 is 50, key is test_row_0/C:col10/1731751201775/Put/seqid=0 2024-11-16T10:00:03,451 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=37267 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=126 2024-11-16T10:00:03,452 DEBUG [RS_FLUSH_OPERATIONS-regionserver/037c58a19ddb:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731751192181.697cd6a363e514bff0d4ec954fc4034b. 2024-11-16T10:00:03,452 INFO [RS_FLUSH_OPERATIONS-regionserver/037c58a19ddb:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HRegion(2902): Flushing 697cd6a363e514bff0d4ec954fc4034b 3/3 column families, dataSize=134.18 KB heapSize=352.31 KB 2024-11-16T10:00:03,456 DEBUG [RS_FLUSH_OPERATIONS-regionserver/037c58a19ddb:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/697cd6a363e514bff0d4ec954fc4034b/.tmp/A/0572be26924348a1ba9c6d41e1d2fc5f is 50, key is test_row_0/A:col10/1731751203024/Put/seqid=0 2024-11-16T10:00:03,459 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37591 is added to blk_1073742306_1482 (size=12357) 2024-11-16T10:00:03,463 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37877 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=125 2024-11-16T10:00:03,466 DEBUG [RS:0;037c58a19ddb:37267-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/697cd6a363e514bff0d4ec954fc4034b/.tmp/C/1eda3944ab0448908f50204d73526559 as hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/697cd6a363e514bff0d4ec954fc4034b/C/1eda3944ab0448908f50204d73526559 2024-11-16T10:00:03,472 INFO [RS:0;037c58a19ddb:37267-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 697cd6a363e514bff0d4ec954fc4034b/C of 697cd6a363e514bff0d4ec954fc4034b into 1eda3944ab0448908f50204d73526559(size=12.1 K), total size for store is 12.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-16T10:00:03,472 DEBUG [RS:0;037c58a19ddb:37267-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 697cd6a363e514bff0d4ec954fc4034b: 2024-11-16T10:00:03,472 INFO [RS:0;037c58a19ddb:37267-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731751192181.697cd6a363e514bff0d4ec954fc4034b., storeName=697cd6a363e514bff0d4ec954fc4034b/C, priority=13, startTime=1731751203414; duration=0sec 2024-11-16T10:00:03,472 DEBUG [RS:0;037c58a19ddb:37267-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-16T10:00:03,472 DEBUG [RS:0;037c58a19ddb:37267-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 697cd6a363e514bff0d4ec954fc4034b:C 2024-11-16T10:00:03,474 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37591 is added to blk_1073742307_1483 (size=12151) 2024-11-16T10:00:03,669 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] regionserver.HRegion(8855): Flush requested on 697cd6a363e514bff0d4ec954fc4034b 2024-11-16T10:00:03,669 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731751192181.697cd6a363e514bff0d4ec954fc4034b. as already flushing 2024-11-16T10:00:03,730 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=697cd6a363e514bff0d4ec954fc4034b, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T10:00:03,731 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36012 deadline: 1731751263720, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=697cd6a363e514bff0d4ec954fc4034b, server=037c58a19ddb,37267,1731751077844 2024-11-16T10:00:03,731 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731751192181.697cd6a363e514bff0d4ec954fc4034b., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 , the old value is region=TestAcidGuarantees,,1731751192181.697cd6a363e514bff0d4ec954fc4034b., hostname=037c58a19ddb,37267,1731751077844, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=697cd6a363e514bff0d4ec954fc4034b, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:00:03,731 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=697cd6a363e514bff0d4ec954fc4034b, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T10:00:03,731 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731751192181.697cd6a363e514bff0d4ec954fc4034b., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=697cd6a363e514bff0d4ec954fc4034b, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:00:03,731 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731751192181.697cd6a363e514bff0d4ec954fc4034b., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 because the exception is null or not the one we care about 2024-11-16T10:00:03,731 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] ipc.CallRunner(138): callId: 81 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36058 deadline: 1731751263720, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=697cd6a363e514bff0d4ec954fc4034b, server=037c58a19ddb,37267,1731751077844 2024-11-16T10:00:03,731 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=697cd6a363e514bff0d4ec954fc4034b, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T10:00:03,731 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731751192181.697cd6a363e514bff0d4ec954fc4034b., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 , the old value is region=TestAcidGuarantees,,1731751192181.697cd6a363e514bff0d4ec954fc4034b., hostname=037c58a19ddb,37267,1731751077844, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=697cd6a363e514bff0d4ec954fc4034b, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:00:03,731 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36022 deadline: 1731751263720, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=697cd6a363e514bff0d4ec954fc4034b, server=037c58a19ddb,37267,1731751077844 2024-11-16T10:00:03,731 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731751192181.697cd6a363e514bff0d4ec954fc4034b., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=697cd6a363e514bff0d4ec954fc4034b, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:00:03,731 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731751192181.697cd6a363e514bff0d4ec954fc4034b., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 because the exception is null or not the one we care about 2024-11-16T10:00:03,732 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731751192181.697cd6a363e514bff0d4ec954fc4034b., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 , the old value is region=TestAcidGuarantees,,1731751192181.697cd6a363e514bff0d4ec954fc4034b., hostname=037c58a19ddb,37267,1731751077844, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=697cd6a363e514bff0d4ec954fc4034b, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:00:03,732 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731751192181.697cd6a363e514bff0d4ec954fc4034b., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=697cd6a363e514bff0d4ec954fc4034b, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:00:03,732 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731751192181.697cd6a363e514bff0d4ec954fc4034b., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 because the exception is null or not the one we care about 2024-11-16T10:00:03,836 DEBUG [RS:0;037c58a19ddb:37267-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/697cd6a363e514bff0d4ec954fc4034b/.tmp/B/173c5e49d87a4489abab8fc2e7bdade3 as hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/697cd6a363e514bff0d4ec954fc4034b/B/173c5e49d87a4489abab8fc2e7bdade3 2024-11-16T10:00:03,840 INFO [RS:0;037c58a19ddb:37267-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 697cd6a363e514bff0d4ec954fc4034b/B of 697cd6a363e514bff0d4ec954fc4034b into 173c5e49d87a4489abab8fc2e7bdade3(size=12.1 K), total size for store is 12.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-16T10:00:03,841 DEBUG [RS:0;037c58a19ddb:37267-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 697cd6a363e514bff0d4ec954fc4034b: 2024-11-16T10:00:03,841 INFO [RS:0;037c58a19ddb:37267-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731751192181.697cd6a363e514bff0d4ec954fc4034b., storeName=697cd6a363e514bff0d4ec954fc4034b/B, priority=13, startTime=1731751203414; duration=0sec 2024-11-16T10:00:03,841 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=697cd6a363e514bff0d4ec954fc4034b, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T10:00:03,841 DEBUG [RS:0;037c58a19ddb:37267-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-16T10:00:03,841 DEBUG [RS:0;037c58a19ddb:37267-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 697cd6a363e514bff0d4ec954fc4034b:B 2024-11-16T10:00:03,841 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36012 deadline: 1731751263833, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=697cd6a363e514bff0d4ec954fc4034b, server=037c58a19ddb,37267,1731751077844 2024-11-16T10:00:03,841 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731751192181.697cd6a363e514bff0d4ec954fc4034b., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 , the old value is region=TestAcidGuarantees,,1731751192181.697cd6a363e514bff0d4ec954fc4034b., hostname=037c58a19ddb,37267,1731751077844, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=697cd6a363e514bff0d4ec954fc4034b, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:00:03,841 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731751192181.697cd6a363e514bff0d4ec954fc4034b., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=697cd6a363e514bff0d4ec954fc4034b, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:00:03,841 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731751192181.697cd6a363e514bff0d4ec954fc4034b., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 because the exception is null or not the one we care about 2024-11-16T10:00:03,852 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=697cd6a363e514bff0d4ec954fc4034b, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T10:00:03,852 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] ipc.CallRunner(138): callId: 82 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36058 deadline: 1731751263843, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=697cd6a363e514bff0d4ec954fc4034b, server=037c58a19ddb,37267,1731751077844 2024-11-16T10:00:03,853 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=697cd6a363e514bff0d4ec954fc4034b, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T10:00:03,853 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] ipc.CallRunner(138): callId: 71 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36022 deadline: 1731751263843, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=697cd6a363e514bff0d4ec954fc4034b, server=037c58a19ddb,37267,1731751077844 2024-11-16T10:00:03,853 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731751192181.697cd6a363e514bff0d4ec954fc4034b., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 , the old value is region=TestAcidGuarantees,,1731751192181.697cd6a363e514bff0d4ec954fc4034b., hostname=037c58a19ddb,37267,1731751077844, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=697cd6a363e514bff0d4ec954fc4034b, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:00:03,853 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731751192181.697cd6a363e514bff0d4ec954fc4034b., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=697cd6a363e514bff0d4ec954fc4034b, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:00:03,853 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731751192181.697cd6a363e514bff0d4ec954fc4034b., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 because the exception is null or not the one we care about 2024-11-16T10:00:03,853 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731751192181.697cd6a363e514bff0d4ec954fc4034b., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 , the old value is region=TestAcidGuarantees,,1731751192181.697cd6a363e514bff0d4ec954fc4034b., hostname=037c58a19ddb,37267,1731751077844, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=697cd6a363e514bff0d4ec954fc4034b, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:00:03,853 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731751192181.697cd6a363e514bff0d4ec954fc4034b., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=697cd6a363e514bff0d4ec954fc4034b, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:00:03,853 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731751192181.697cd6a363e514bff0d4ec954fc4034b., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 because the exception is null or not the one we care about 2024-11-16T10:00:03,880 INFO [RS_FLUSH_OPERATIONS-regionserver/037c58a19ddb:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=192 (bloomFilter=true), to=hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/697cd6a363e514bff0d4ec954fc4034b/.tmp/A/0572be26924348a1ba9c6d41e1d2fc5f 2024-11-16T10:00:03,887 DEBUG [RS_FLUSH_OPERATIONS-regionserver/037c58a19ddb:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/697cd6a363e514bff0d4ec954fc4034b/.tmp/B/0a22ef23afe44699a9b457690504a95f is 50, key is test_row_0/B:col10/1731751203024/Put/seqid=0 2024-11-16T10:00:03,890 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37591 is added to blk_1073742308_1484 (size=12151) 2024-11-16T10:00:03,972 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37877 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=125 2024-11-16T10:00:04,047 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=697cd6a363e514bff0d4ec954fc4034b, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T10:00:04,047 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] ipc.CallRunner(138): callId: 71 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36012 deadline: 1731751264043, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=697cd6a363e514bff0d4ec954fc4034b, server=037c58a19ddb,37267,1731751077844 2024-11-16T10:00:04,047 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731751192181.697cd6a363e514bff0d4ec954fc4034b., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 , the old value is region=TestAcidGuarantees,,1731751192181.697cd6a363e514bff0d4ec954fc4034b., hostname=037c58a19ddb,37267,1731751077844, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=697cd6a363e514bff0d4ec954fc4034b, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:00:04,047 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731751192181.697cd6a363e514bff0d4ec954fc4034b., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=697cd6a363e514bff0d4ec954fc4034b, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:00:04,047 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731751192181.697cd6a363e514bff0d4ec954fc4034b., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 because the exception is null or not the one we care about 2024-11-16T10:00:04,068 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=697cd6a363e514bff0d4ec954fc4034b, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T10:00:04,068 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] ipc.CallRunner(138): callId: 83 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36058 deadline: 1731751264062, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=697cd6a363e514bff0d4ec954fc4034b, server=037c58a19ddb,37267,1731751077844 2024-11-16T10:00:04,068 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=697cd6a363e514bff0d4ec954fc4034b, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T10:00:04,068 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731751192181.697cd6a363e514bff0d4ec954fc4034b., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 , the old value is region=TestAcidGuarantees,,1731751192181.697cd6a363e514bff0d4ec954fc4034b., hostname=037c58a19ddb,37267,1731751077844, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=697cd6a363e514bff0d4ec954fc4034b, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:00:04,068 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] ipc.CallRunner(138): callId: 72 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36022 deadline: 1731751264062, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=697cd6a363e514bff0d4ec954fc4034b, server=037c58a19ddb,37267,1731751077844 2024-11-16T10:00:04,068 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731751192181.697cd6a363e514bff0d4ec954fc4034b., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=697cd6a363e514bff0d4ec954fc4034b, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:00:04,068 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731751192181.697cd6a363e514bff0d4ec954fc4034b., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 because the exception is null or not the one we care about 2024-11-16T10:00:04,068 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731751192181.697cd6a363e514bff0d4ec954fc4034b., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 , the old value is region=TestAcidGuarantees,,1731751192181.697cd6a363e514bff0d4ec954fc4034b., hostname=037c58a19ddb,37267,1731751077844, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=697cd6a363e514bff0d4ec954fc4034b, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:00:04,069 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731751192181.697cd6a363e514bff0d4ec954fc4034b., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=697cd6a363e514bff0d4ec954fc4034b, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:00:04,069 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731751192181.697cd6a363e514bff0d4ec954fc4034b., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 because the exception is null or not the one we care about 2024-11-16T10:00:04,291 INFO [RS_FLUSH_OPERATIONS-regionserver/037c58a19ddb:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=192 (bloomFilter=true), to=hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/697cd6a363e514bff0d4ec954fc4034b/.tmp/B/0a22ef23afe44699a9b457690504a95f 2024-11-16T10:00:04,298 DEBUG [RS_FLUSH_OPERATIONS-regionserver/037c58a19ddb:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/697cd6a363e514bff0d4ec954fc4034b/.tmp/C/db2e5ea5d8c048eaa89b7e96a9c2e23a is 50, key is test_row_0/C:col10/1731751203024/Put/seqid=0 2024-11-16T10:00:04,302 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37591 is added to blk_1073742309_1485 (size=12151) 2024-11-16T10:00:04,356 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=697cd6a363e514bff0d4ec954fc4034b, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T10:00:04,356 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] ipc.CallRunner(138): callId: 72 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36012 deadline: 1731751264353, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=697cd6a363e514bff0d4ec954fc4034b, server=037c58a19ddb,37267,1731751077844 2024-11-16T10:00:04,356 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731751192181.697cd6a363e514bff0d4ec954fc4034b., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 , the old value is region=TestAcidGuarantees,,1731751192181.697cd6a363e514bff0d4ec954fc4034b., hostname=037c58a19ddb,37267,1731751077844, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=697cd6a363e514bff0d4ec954fc4034b, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:00:04,356 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731751192181.697cd6a363e514bff0d4ec954fc4034b., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=697cd6a363e514bff0d4ec954fc4034b, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:00:04,356 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731751192181.697cd6a363e514bff0d4ec954fc4034b., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 because the exception is null or not the one we care about 2024-11-16T10:00:04,377 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=697cd6a363e514bff0d4ec954fc4034b, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T10:00:04,377 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] ipc.CallRunner(138): callId: 84 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36058 deadline: 1731751264373, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=697cd6a363e514bff0d4ec954fc4034b, server=037c58a19ddb,37267,1731751077844 2024-11-16T10:00:04,377 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=697cd6a363e514bff0d4ec954fc4034b, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T10:00:04,378 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731751192181.697cd6a363e514bff0d4ec954fc4034b., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 , the old value is region=TestAcidGuarantees,,1731751192181.697cd6a363e514bff0d4ec954fc4034b., hostname=037c58a19ddb,37267,1731751077844, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=697cd6a363e514bff0d4ec954fc4034b, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:00:04,378 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] ipc.CallRunner(138): callId: 73 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36022 deadline: 1731751264373, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=697cd6a363e514bff0d4ec954fc4034b, server=037c58a19ddb,37267,1731751077844 2024-11-16T10:00:04,378 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731751192181.697cd6a363e514bff0d4ec954fc4034b., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=697cd6a363e514bff0d4ec954fc4034b, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:00:04,378 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731751192181.697cd6a363e514bff0d4ec954fc4034b., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 because the exception is null or not the one we care about 2024-11-16T10:00:04,378 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731751192181.697cd6a363e514bff0d4ec954fc4034b., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 , the old value is region=TestAcidGuarantees,,1731751192181.697cd6a363e514bff0d4ec954fc4034b., hostname=037c58a19ddb,37267,1731751077844, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=697cd6a363e514bff0d4ec954fc4034b, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:00:04,378 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731751192181.697cd6a363e514bff0d4ec954fc4034b., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=697cd6a363e514bff0d4ec954fc4034b, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:00:04,378 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731751192181.697cd6a363e514bff0d4ec954fc4034b., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 because the exception is null or not the one we care about 2024-11-16T10:00:04,703 INFO [RS_FLUSH_OPERATIONS-regionserver/037c58a19ddb:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=192 (bloomFilter=true), to=hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/697cd6a363e514bff0d4ec954fc4034b/.tmp/C/db2e5ea5d8c048eaa89b7e96a9c2e23a 2024-11-16T10:00:04,707 DEBUG [RS_FLUSH_OPERATIONS-regionserver/037c58a19ddb:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/697cd6a363e514bff0d4ec954fc4034b/.tmp/A/0572be26924348a1ba9c6d41e1d2fc5f as hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/697cd6a363e514bff0d4ec954fc4034b/A/0572be26924348a1ba9c6d41e1d2fc5f 2024-11-16T10:00:04,711 INFO [RS_FLUSH_OPERATIONS-regionserver/037c58a19ddb:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/697cd6a363e514bff0d4ec954fc4034b/A/0572be26924348a1ba9c6d41e1d2fc5f, entries=150, sequenceid=192, filesize=11.9 K 2024-11-16T10:00:04,712 DEBUG [RS_FLUSH_OPERATIONS-regionserver/037c58a19ddb:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/697cd6a363e514bff0d4ec954fc4034b/.tmp/B/0a22ef23afe44699a9b457690504a95f as hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/697cd6a363e514bff0d4ec954fc4034b/B/0a22ef23afe44699a9b457690504a95f 2024-11-16T10:00:04,716 INFO [RS_FLUSH_OPERATIONS-regionserver/037c58a19ddb:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/697cd6a363e514bff0d4ec954fc4034b/B/0a22ef23afe44699a9b457690504a95f, entries=150, sequenceid=192, filesize=11.9 K 2024-11-16T10:00:04,719 DEBUG [RS_FLUSH_OPERATIONS-regionserver/037c58a19ddb:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/697cd6a363e514bff0d4ec954fc4034b/.tmp/C/db2e5ea5d8c048eaa89b7e96a9c2e23a as hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/697cd6a363e514bff0d4ec954fc4034b/C/db2e5ea5d8c048eaa89b7e96a9c2e23a 2024-11-16T10:00:04,723 INFO [RS_FLUSH_OPERATIONS-regionserver/037c58a19ddb:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/697cd6a363e514bff0d4ec954fc4034b/C/db2e5ea5d8c048eaa89b7e96a9c2e23a, entries=150, sequenceid=192, filesize=11.9 K 2024-11-16T10:00:04,724 INFO [RS_FLUSH_OPERATIONS-regionserver/037c58a19ddb:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HRegion(3140): Finished flush of dataSize ~134.18 KB/137400, heapSize ~352.27 KB/360720, currentSize=67.09 KB/68700 for 697cd6a363e514bff0d4ec954fc4034b in 1272ms, sequenceid=192, compaction requested=false 2024-11-16T10:00:04,724 DEBUG [RS_FLUSH_OPERATIONS-regionserver/037c58a19ddb:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HRegion(2603): Flush status journal for 697cd6a363e514bff0d4ec954fc4034b: 2024-11-16T10:00:04,724 DEBUG [RS_FLUSH_OPERATIONS-regionserver/037c58a19ddb:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731751192181.697cd6a363e514bff0d4ec954fc4034b. 2024-11-16T10:00:04,724 DEBUG [RS_FLUSH_OPERATIONS-regionserver/037c58a19ddb:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=126 2024-11-16T10:00:04,725 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37877 {}] master.HMaster(4169): Remote procedure done, pid=126 2024-11-16T10:00:04,728 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=126, resume processing ppid=125 2024-11-16T10:00:04,728 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=126, ppid=125, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.8890 sec 2024-11-16T10:00:04,731 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=125, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=125, table=TestAcidGuarantees in 1.8950 sec 2024-11-16T10:00:04,867 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] regionserver.HRegion(8855): Flush requested on 697cd6a363e514bff0d4ec954fc4034b 2024-11-16T10:00:04,867 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-16T10:00:04,867 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 697cd6a363e514bff0d4ec954fc4034b 3/3 column families, dataSize=73.80 KB heapSize=194.11 KB 2024-11-16T10:00:04,872 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/697cd6a363e514bff0d4ec954fc4034b/.tmp/A/6fd014921977453799c306ea60a1a1ef is 50, key is test_row_0/A:col10/1731751203690/Put/seqid=0 2024-11-16T10:00:04,875 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37591 is added to blk_1073742310_1486 (size=14541) 2024-11-16T10:00:04,877 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=208 (bloomFilter=true), to=hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/697cd6a363e514bff0d4ec954fc4034b/.tmp/A/6fd014921977453799c306ea60a1a1ef 2024-11-16T10:00:04,884 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/697cd6a363e514bff0d4ec954fc4034b/.tmp/B/5ebf14cd6b1c424d9087c461c3fefa86 is 50, key is test_row_0/B:col10/1731751203690/Put/seqid=0 2024-11-16T10:00:04,892 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37591 is added to blk_1073742311_1487 (size=12151) 2024-11-16T10:00:04,894 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=208 (bloomFilter=true), to=hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/697cd6a363e514bff0d4ec954fc4034b/.tmp/B/5ebf14cd6b1c424d9087c461c3fefa86 2024-11-16T10:00:04,902 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/697cd6a363e514bff0d4ec954fc4034b/.tmp/C/4c12439cbf7a45158cddfb8f850956bf is 50, key is test_row_0/C:col10/1731751203690/Put/seqid=0 2024-11-16T10:00:04,908 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37591 is added to blk_1073742312_1488 (size=12151) 2024-11-16T10:00:04,982 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37877 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=125 2024-11-16T10:00:04,983 INFO [RPCClient-NioEventLoopGroup-4-10 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-16T10:00:04,983 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37877 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-16T10:00:04,984 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37877 {}] procedure2.ProcedureExecutor(1139): Stored pid=127, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=127, table=TestAcidGuarantees 2024-11-16T10:00:04,985 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37877 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=127 2024-11-16T10:00:04,985 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=127, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=127, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-16T10:00:04,986 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=127, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=127, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-16T10:00:04,986 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=128, ppid=127, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-16T10:00:04,994 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=697cd6a363e514bff0d4ec954fc4034b, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T10:00:04,994 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36046 deadline: 1731751264963, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=697cd6a363e514bff0d4ec954fc4034b, server=037c58a19ddb,37267,1731751077844 2024-11-16T10:00:04,994 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=697cd6a363e514bff0d4ec954fc4034b, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T10:00:04,995 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] ipc.CallRunner(138): callId: 79 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36022 deadline: 1731751264982, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=697cd6a363e514bff0d4ec954fc4034b, server=037c58a19ddb,37267,1731751077844 2024-11-16T10:00:04,995 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=697cd6a363e514bff0d4ec954fc4034b, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T10:00:04,995 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] ipc.CallRunner(138): callId: 82 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36012 deadline: 1731751264982, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=697cd6a363e514bff0d4ec954fc4034b, server=037c58a19ddb,37267,1731751077844 2024-11-16T10:00:04,994 WARN [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRpcRetryingCaller(177): Call to 037c58a19ddb,37267,1731751077844 for 'test_row_2' in 697cd6a363e514bff0d4ec954fc4034b of TestAcidGuarantees failed, tries = 8, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 8281 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=697cd6a363e514bff0d4ec954fc4034b, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor44.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=697cd6a363e514bff0d4ec954fc4034b, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-16T10:00:04,995 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731751192181.697cd6a363e514bff0d4ec954fc4034b., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 , the old value is region=TestAcidGuarantees,,1731751192181.697cd6a363e514bff0d4ec954fc4034b., hostname=037c58a19ddb,37267,1731751077844, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=697cd6a363e514bff0d4ec954fc4034b, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:00:04,995 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731751192181.697cd6a363e514bff0d4ec954fc4034b., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=697cd6a363e514bff0d4ec954fc4034b, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:00:04,995 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=697cd6a363e514bff0d4ec954fc4034b, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T10:00:04,995 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731751192181.697cd6a363e514bff0d4ec954fc4034b., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 because the exception is null or not the one we care about 2024-11-16T10:00:04,995 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] ipc.CallRunner(138): callId: 90 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36058 deadline: 1731751264982, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=697cd6a363e514bff0d4ec954fc4034b, server=037c58a19ddb,37267,1731751077844 2024-11-16T10:00:04,995 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731751192181.697cd6a363e514bff0d4ec954fc4034b., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 , the old value is region=TestAcidGuarantees,,1731751192181.697cd6a363e514bff0d4ec954fc4034b., hostname=037c58a19ddb,37267,1731751077844, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=697cd6a363e514bff0d4ec954fc4034b, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:00:04,995 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731751192181.697cd6a363e514bff0d4ec954fc4034b., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=697cd6a363e514bff0d4ec954fc4034b, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:00:04,996 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731751192181.697cd6a363e514bff0d4ec954fc4034b., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 because the exception is null or not the one we care about 2024-11-16T10:00:04,996 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731751192181.697cd6a363e514bff0d4ec954fc4034b., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 , the old value is region=TestAcidGuarantees,,1731751192181.697cd6a363e514bff0d4ec954fc4034b., hostname=037c58a19ddb,37267,1731751077844, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=697cd6a363e514bff0d4ec954fc4034b, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:00:04,996 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731751192181.697cd6a363e514bff0d4ec954fc4034b., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=697cd6a363e514bff0d4ec954fc4034b, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:00:04,996 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731751192181.697cd6a363e514bff0d4ec954fc4034b., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 because the exception is null or not the one we care about 2024-11-16T10:00:04,996 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731751192181.697cd6a363e514bff0d4ec954fc4034b., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 , the old value is region=TestAcidGuarantees,,1731751192181.697cd6a363e514bff0d4ec954fc4034b., hostname=037c58a19ddb,37267,1731751077844, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=697cd6a363e514bff0d4ec954fc4034b, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:00:04,996 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731751192181.697cd6a363e514bff0d4ec954fc4034b., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=697cd6a363e514bff0d4ec954fc4034b, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:00:04,996 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731751192181.697cd6a363e514bff0d4ec954fc4034b., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 because the exception is null or not the one we care about 2024-11-16T10:00:05,003 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=697cd6a363e514bff0d4ec954fc4034b, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T10:00:05,003 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36086 deadline: 1731751264993, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=697cd6a363e514bff0d4ec954fc4034b, server=037c58a19ddb,37267,1731751077844 2024-11-16T10:00:05,003 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731751192181.697cd6a363e514bff0d4ec954fc4034b., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 , the old value is region=TestAcidGuarantees,,1731751192181.697cd6a363e514bff0d4ec954fc4034b., hostname=037c58a19ddb,37267,1731751077844, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=697cd6a363e514bff0d4ec954fc4034b, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:00:05,003 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731751192181.697cd6a363e514bff0d4ec954fc4034b., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=697cd6a363e514bff0d4ec954fc4034b, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:00:05,003 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731751192181.697cd6a363e514bff0d4ec954fc4034b., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 because the exception is null or not the one we care about 2024-11-16T10:00:05,092 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37877 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=127 2024-11-16T10:00:05,111 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=697cd6a363e514bff0d4ec954fc4034b, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T10:00:05,111 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] ipc.CallRunner(138): callId: 91 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36058 deadline: 1731751265102, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=697cd6a363e514bff0d4ec954fc4034b, server=037c58a19ddb,37267,1731751077844 2024-11-16T10:00:05,111 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=697cd6a363e514bff0d4ec954fc4034b, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T10:00:05,111 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] ipc.CallRunner(138): callId: 80 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36022 deadline: 1731751265103, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=697cd6a363e514bff0d4ec954fc4034b, server=037c58a19ddb,37267,1731751077844 2024-11-16T10:00:05,112 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731751192181.697cd6a363e514bff0d4ec954fc4034b., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 , the old value is region=TestAcidGuarantees,,1731751192181.697cd6a363e514bff0d4ec954fc4034b., hostname=037c58a19ddb,37267,1731751077844, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=697cd6a363e514bff0d4ec954fc4034b, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:00:05,112 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731751192181.697cd6a363e514bff0d4ec954fc4034b., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=697cd6a363e514bff0d4ec954fc4034b, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:00:05,112 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731751192181.697cd6a363e514bff0d4ec954fc4034b., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 because the exception is null or not the one we care about 2024-11-16T10:00:05,112 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=697cd6a363e514bff0d4ec954fc4034b, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T10:00:05,112 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731751192181.697cd6a363e514bff0d4ec954fc4034b., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 , the old value is region=TestAcidGuarantees,,1731751192181.697cd6a363e514bff0d4ec954fc4034b., hostname=037c58a19ddb,37267,1731751077844, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=697cd6a363e514bff0d4ec954fc4034b, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:00:05,112 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731751192181.697cd6a363e514bff0d4ec954fc4034b., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=697cd6a363e514bff0d4ec954fc4034b, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:00:05,112 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731751192181.697cd6a363e514bff0d4ec954fc4034b., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 because the exception is null or not the one we care about 2024-11-16T10:00:05,112 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] ipc.CallRunner(138): callId: 83 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36012 deadline: 1731751265103, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=697cd6a363e514bff0d4ec954fc4034b, server=037c58a19ddb,37267,1731751077844 2024-11-16T10:00:05,112 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731751192181.697cd6a363e514bff0d4ec954fc4034b., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 , the old value is region=TestAcidGuarantees,,1731751192181.697cd6a363e514bff0d4ec954fc4034b., hostname=037c58a19ddb,37267,1731751077844, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=697cd6a363e514bff0d4ec954fc4034b, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:00:05,112 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731751192181.697cd6a363e514bff0d4ec954fc4034b., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=697cd6a363e514bff0d4ec954fc4034b, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:00:05,112 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731751192181.697cd6a363e514bff0d4ec954fc4034b., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 because the exception is null or not the one we care about 2024-11-16T10:00:05,123 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=697cd6a363e514bff0d4ec954fc4034b, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T10:00:05,123 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36086 deadline: 1731751265113, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=697cd6a363e514bff0d4ec954fc4034b, server=037c58a19ddb,37267,1731751077844 2024-11-16T10:00:05,123 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731751192181.697cd6a363e514bff0d4ec954fc4034b., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 , the old value is region=TestAcidGuarantees,,1731751192181.697cd6a363e514bff0d4ec954fc4034b., hostname=037c58a19ddb,37267,1731751077844, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=697cd6a363e514bff0d4ec954fc4034b, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:00:05,123 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731751192181.697cd6a363e514bff0d4ec954fc4034b., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=697cd6a363e514bff0d4ec954fc4034b, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:00:05,123 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731751192181.697cd6a363e514bff0d4ec954fc4034b., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 because the exception is null or not the one we care about 2024-11-16T10:00:05,138 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=37267 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=128 2024-11-16T10:00:05,138 DEBUG [RS_FLUSH_OPERATIONS-regionserver/037c58a19ddb:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731751192181.697cd6a363e514bff0d4ec954fc4034b. 2024-11-16T10:00:05,138 DEBUG [RS_FLUSH_OPERATIONS-regionserver/037c58a19ddb:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731751192181.697cd6a363e514bff0d4ec954fc4034b. as already flushing 2024-11-16T10:00:05,138 DEBUG [RS_FLUSH_OPERATIONS-regionserver/037c58a19ddb:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731751192181.697cd6a363e514bff0d4ec954fc4034b. 2024-11-16T10:00:05,138 ERROR [RS_FLUSH_OPERATIONS-regionserver/037c58a19ddb:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] handler.RSProcedureHandler(58): pid=128 java.io.IOException: Unable to complete flush {ENCODED => 697cd6a363e514bff0d4ec954fc4034b, NAME => 'TestAcidGuarantees,,1731751192181.697cd6a363e514bff0d4ec954fc4034b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T10:00:05,139 DEBUG [RS_FLUSH_OPERATIONS-regionserver/037c58a19ddb:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=128 java.io.IOException: Unable to complete flush {ENCODED => 697cd6a363e514bff0d4ec954fc4034b, NAME => 'TestAcidGuarantees,,1731751192181.697cd6a363e514bff0d4ec954fc4034b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T10:00:05,139 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37877 {}] master.HMaster(4177): Remote procedure failed, pid=128 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 697cd6a363e514bff0d4ec954fc4034b, NAME => 'TestAcidGuarantees,,1731751192181.697cd6a363e514bff0d4ec954fc4034b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 697cd6a363e514bff0d4ec954fc4034b, NAME => 'TestAcidGuarantees,,1731751192181.697cd6a363e514bff0d4ec954fc4034b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T10:00:05,291 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=37267 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=128 2024-11-16T10:00:05,292 DEBUG [RS_FLUSH_OPERATIONS-regionserver/037c58a19ddb:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731751192181.697cd6a363e514bff0d4ec954fc4034b. 2024-11-16T10:00:05,292 DEBUG [RS_FLUSH_OPERATIONS-regionserver/037c58a19ddb:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731751192181.697cd6a363e514bff0d4ec954fc4034b. as already flushing 2024-11-16T10:00:05,292 DEBUG [RS_FLUSH_OPERATIONS-regionserver/037c58a19ddb:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731751192181.697cd6a363e514bff0d4ec954fc4034b. 2024-11-16T10:00:05,292 ERROR [RS_FLUSH_OPERATIONS-regionserver/037c58a19ddb:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] handler.RSProcedureHandler(58): pid=128 java.io.IOException: Unable to complete flush {ENCODED => 697cd6a363e514bff0d4ec954fc4034b, NAME => 'TestAcidGuarantees,,1731751192181.697cd6a363e514bff0d4ec954fc4034b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T10:00:05,292 DEBUG [RS_FLUSH_OPERATIONS-regionserver/037c58a19ddb:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=128 java.io.IOException: Unable to complete flush {ENCODED => 697cd6a363e514bff0d4ec954fc4034b, NAME => 'TestAcidGuarantees,,1731751192181.697cd6a363e514bff0d4ec954fc4034b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T10:00:05,293 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37877 {}] master.HMaster(4177): Remote procedure failed, pid=128 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 697cd6a363e514bff0d4ec954fc4034b, NAME => 'TestAcidGuarantees,,1731751192181.697cd6a363e514bff0d4ec954fc4034b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 697cd6a363e514bff0d4ec954fc4034b, NAME => 'TestAcidGuarantees,,1731751192181.697cd6a363e514bff0d4ec954fc4034b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T10:00:05,302 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37877 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=127 2024-11-16T10:00:05,308 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=208 (bloomFilter=true), to=hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/697cd6a363e514bff0d4ec954fc4034b/.tmp/C/4c12439cbf7a45158cddfb8f850956bf 2024-11-16T10:00:05,312 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/697cd6a363e514bff0d4ec954fc4034b/.tmp/A/6fd014921977453799c306ea60a1a1ef as hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/697cd6a363e514bff0d4ec954fc4034b/A/6fd014921977453799c306ea60a1a1ef 2024-11-16T10:00:05,316 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/697cd6a363e514bff0d4ec954fc4034b/A/6fd014921977453799c306ea60a1a1ef, entries=200, sequenceid=208, filesize=14.2 K 2024-11-16T10:00:05,316 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/697cd6a363e514bff0d4ec954fc4034b/.tmp/B/5ebf14cd6b1c424d9087c461c3fefa86 as hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/697cd6a363e514bff0d4ec954fc4034b/B/5ebf14cd6b1c424d9087c461c3fefa86 2024-11-16T10:00:05,319 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/697cd6a363e514bff0d4ec954fc4034b/B/5ebf14cd6b1c424d9087c461c3fefa86, entries=150, sequenceid=208, filesize=11.9 K 2024-11-16T10:00:05,320 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/697cd6a363e514bff0d4ec954fc4034b/.tmp/C/4c12439cbf7a45158cddfb8f850956bf as hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/697cd6a363e514bff0d4ec954fc4034b/C/4c12439cbf7a45158cddfb8f850956bf 2024-11-16T10:00:05,325 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/697cd6a363e514bff0d4ec954fc4034b/C/4c12439cbf7a45158cddfb8f850956bf, entries=150, sequenceid=208, filesize=11.9 K 2024-11-16T10:00:05,326 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~73.80 KB/75570, heapSize ~194.06 KB/198720, currentSize=127.47 KB/130530 for 697cd6a363e514bff0d4ec954fc4034b in 459ms, sequenceid=208, compaction requested=true 2024-11-16T10:00:05,326 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 697cd6a363e514bff0d4ec954fc4034b: 2024-11-16T10:00:05,326 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 697cd6a363e514bff0d4ec954fc4034b:A, priority=-2147483648, current under compaction store size is 1 2024-11-16T10:00:05,327 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-16T10:00:05,327 DEBUG [RS:0;037c58a19ddb:37267-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-16T10:00:05,327 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 697cd6a363e514bff0d4ec954fc4034b:B, priority=-2147483648, current under compaction store size is 2 2024-11-16T10:00:05,327 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-16T10:00:05,327 DEBUG [RS:0;037c58a19ddb:37267-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-16T10:00:05,327 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 697cd6a363e514bff0d4ec954fc4034b:C, priority=-2147483648, current under compaction store size is 3 2024-11-16T10:00:05,327 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-16T10:00:05,327 DEBUG [RS:0;037c58a19ddb:37267-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36659 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-16T10:00:05,327 DEBUG [RS:0;037c58a19ddb:37267-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 39049 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-16T10:00:05,327 DEBUG [RS:0;037c58a19ddb:37267-longCompactions-0 {}] regionserver.HStore(1541): 697cd6a363e514bff0d4ec954fc4034b/B is initiating minor compaction (all files) 2024-11-16T10:00:05,327 DEBUG [RS:0;037c58a19ddb:37267-shortCompactions-0 {}] regionserver.HStore(1541): 697cd6a363e514bff0d4ec954fc4034b/A is initiating minor compaction (all files) 2024-11-16T10:00:05,328 INFO [RS:0;037c58a19ddb:37267-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 697cd6a363e514bff0d4ec954fc4034b/B in TestAcidGuarantees,,1731751192181.697cd6a363e514bff0d4ec954fc4034b. 2024-11-16T10:00:05,328 INFO [RS:0;037c58a19ddb:37267-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 697cd6a363e514bff0d4ec954fc4034b/A in TestAcidGuarantees,,1731751192181.697cd6a363e514bff0d4ec954fc4034b. 2024-11-16T10:00:05,328 INFO [RS:0;037c58a19ddb:37267-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/697cd6a363e514bff0d4ec954fc4034b/B/173c5e49d87a4489abab8fc2e7bdade3, hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/697cd6a363e514bff0d4ec954fc4034b/B/0a22ef23afe44699a9b457690504a95f, hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/697cd6a363e514bff0d4ec954fc4034b/B/5ebf14cd6b1c424d9087c461c3fefa86] into tmpdir=hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/697cd6a363e514bff0d4ec954fc4034b/.tmp, totalSize=35.8 K 2024-11-16T10:00:05,328 INFO [RS:0;037c58a19ddb:37267-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/697cd6a363e514bff0d4ec954fc4034b/A/387791a6a1904f44b3973f11b149c72a, hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/697cd6a363e514bff0d4ec954fc4034b/A/0572be26924348a1ba9c6d41e1d2fc5f, hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/697cd6a363e514bff0d4ec954fc4034b/A/6fd014921977453799c306ea60a1a1ef] into tmpdir=hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/697cd6a363e514bff0d4ec954fc4034b/.tmp, totalSize=38.1 K 2024-11-16T10:00:05,328 DEBUG [RS:0;037c58a19ddb:37267-longCompactions-0 {}] compactions.Compactor(225): Compacting 173c5e49d87a4489abab8fc2e7bdade3, keycount=150, bloomtype=ROW, size=12.1 K, encoding=NONE, compression=NONE, seqNum=168, earliestPutTs=1731751201775 2024-11-16T10:00:05,328 DEBUG [RS:0;037c58a19ddb:37267-shortCompactions-0 {}] compactions.Compactor(225): Compacting 387791a6a1904f44b3973f11b149c72a, keycount=150, bloomtype=ROW, size=12.1 K, encoding=NONE, compression=NONE, seqNum=168, earliestPutTs=1731751201775 2024-11-16T10:00:05,328 DEBUG [RS:0;037c58a19ddb:37267-longCompactions-0 {}] compactions.Compactor(225): Compacting 0a22ef23afe44699a9b457690504a95f, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=192, earliestPutTs=1731751203013 2024-11-16T10:00:05,328 DEBUG [RS:0;037c58a19ddb:37267-shortCompactions-0 {}] compactions.Compactor(225): Compacting 0572be26924348a1ba9c6d41e1d2fc5f, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=192, earliestPutTs=1731751203013 2024-11-16T10:00:05,329 DEBUG [RS:0;037c58a19ddb:37267-shortCompactions-0 {}] compactions.Compactor(225): Compacting 6fd014921977453799c306ea60a1a1ef, keycount=200, bloomtype=ROW, size=14.2 K, encoding=NONE, compression=NONE, seqNum=208, earliestPutTs=1731751203690 2024-11-16T10:00:05,329 DEBUG [RS:0;037c58a19ddb:37267-longCompactions-0 {}] compactions.Compactor(225): Compacting 5ebf14cd6b1c424d9087c461c3fefa86, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=208, earliestPutTs=1731751203690 2024-11-16T10:00:05,334 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] regionserver.HRegion(8855): Flush requested on 697cd6a363e514bff0d4ec954fc4034b 2024-11-16T10:00:05,334 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-16T10:00:05,334 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 697cd6a363e514bff0d4ec954fc4034b 3/3 column families, dataSize=147.60 KB heapSize=387.47 KB 2024-11-16T10:00:05,336 INFO [RS:0;037c58a19ddb:37267-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 697cd6a363e514bff0d4ec954fc4034b#B#compaction#415 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-16T10:00:05,336 INFO [RS:0;037c58a19ddb:37267-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 697cd6a363e514bff0d4ec954fc4034b#A#compaction#416 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-16T10:00:05,336 DEBUG [RS:0;037c58a19ddb:37267-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/697cd6a363e514bff0d4ec954fc4034b/.tmp/B/090ecb975fe84165b550dd5f4def4066 is 50, key is test_row_0/B:col10/1731751203690/Put/seqid=0 2024-11-16T10:00:05,336 DEBUG [RS:0;037c58a19ddb:37267-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/697cd6a363e514bff0d4ec954fc4034b/.tmp/A/aa422c222ac1447d8325f707092eaf7e is 50, key is test_row_0/A:col10/1731751203690/Put/seqid=0 2024-11-16T10:00:05,340 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37591 is added to blk_1073742313_1489 (size=12459) 2024-11-16T10:00:05,347 DEBUG [RS:0;037c58a19ddb:37267-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/697cd6a363e514bff0d4ec954fc4034b/.tmp/B/090ecb975fe84165b550dd5f4def4066 as hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/697cd6a363e514bff0d4ec954fc4034b/B/090ecb975fe84165b550dd5f4def4066 2024-11-16T10:00:05,351 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37591 is added to blk_1073742314_1490 (size=12459) 2024-11-16T10:00:05,351 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/697cd6a363e514bff0d4ec954fc4034b/.tmp/A/3cecc1d82dd343439dbccdd090ece5fd is 50, key is test_row_0/A:col10/1731751204930/Put/seqid=0 2024-11-16T10:00:05,356 DEBUG [RS:0;037c58a19ddb:37267-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/697cd6a363e514bff0d4ec954fc4034b/.tmp/A/aa422c222ac1447d8325f707092eaf7e as hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/697cd6a363e514bff0d4ec954fc4034b/A/aa422c222ac1447d8325f707092eaf7e 2024-11-16T10:00:05,356 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37591 is added to blk_1073742315_1491 (size=12151) 2024-11-16T10:00:05,357 INFO [RS:0;037c58a19ddb:37267-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 697cd6a363e514bff0d4ec954fc4034b/B of 697cd6a363e514bff0d4ec954fc4034b into 090ecb975fe84165b550dd5f4def4066(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-16T10:00:05,357 DEBUG [RS:0;037c58a19ddb:37267-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 697cd6a363e514bff0d4ec954fc4034b: 2024-11-16T10:00:05,357 INFO [RS:0;037c58a19ddb:37267-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731751192181.697cd6a363e514bff0d4ec954fc4034b., storeName=697cd6a363e514bff0d4ec954fc4034b/B, priority=13, startTime=1731751205327; duration=0sec 2024-11-16T10:00:05,357 DEBUG [RS:0;037c58a19ddb:37267-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-16T10:00:05,357 DEBUG [RS:0;037c58a19ddb:37267-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 697cd6a363e514bff0d4ec954fc4034b:B 2024-11-16T10:00:05,357 DEBUG [RS:0;037c58a19ddb:37267-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-16T10:00:05,358 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=233 (bloomFilter=true), to=hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/697cd6a363e514bff0d4ec954fc4034b/.tmp/A/3cecc1d82dd343439dbccdd090ece5fd 2024-11-16T10:00:05,359 DEBUG [RS:0;037c58a19ddb:37267-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36659 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-16T10:00:05,359 DEBUG [RS:0;037c58a19ddb:37267-longCompactions-0 {}] regionserver.HStore(1541): 697cd6a363e514bff0d4ec954fc4034b/C is initiating minor compaction (all files) 2024-11-16T10:00:05,359 INFO [RS:0;037c58a19ddb:37267-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 697cd6a363e514bff0d4ec954fc4034b/C in TestAcidGuarantees,,1731751192181.697cd6a363e514bff0d4ec954fc4034b. 2024-11-16T10:00:05,359 INFO [RS:0;037c58a19ddb:37267-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/697cd6a363e514bff0d4ec954fc4034b/C/1eda3944ab0448908f50204d73526559, hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/697cd6a363e514bff0d4ec954fc4034b/C/db2e5ea5d8c048eaa89b7e96a9c2e23a, hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/697cd6a363e514bff0d4ec954fc4034b/C/4c12439cbf7a45158cddfb8f850956bf] into tmpdir=hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/697cd6a363e514bff0d4ec954fc4034b/.tmp, totalSize=35.8 K 2024-11-16T10:00:05,360 DEBUG [RS:0;037c58a19ddb:37267-longCompactions-0 {}] compactions.Compactor(225): Compacting 1eda3944ab0448908f50204d73526559, keycount=150, bloomtype=ROW, size=12.1 K, encoding=NONE, compression=NONE, seqNum=168, earliestPutTs=1731751201775 2024-11-16T10:00:05,360 DEBUG [RS:0;037c58a19ddb:37267-longCompactions-0 {}] compactions.Compactor(225): Compacting db2e5ea5d8c048eaa89b7e96a9c2e23a, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=192, earliestPutTs=1731751203013 2024-11-16T10:00:05,361 DEBUG [RS:0;037c58a19ddb:37267-longCompactions-0 {}] compactions.Compactor(225): Compacting 4c12439cbf7a45158cddfb8f850956bf, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=208, earliestPutTs=1731751203690 2024-11-16T10:00:05,363 INFO [RS:0;037c58a19ddb:37267-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 697cd6a363e514bff0d4ec954fc4034b/A of 697cd6a363e514bff0d4ec954fc4034b into aa422c222ac1447d8325f707092eaf7e(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-16T10:00:05,363 DEBUG [RS:0;037c58a19ddb:37267-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 697cd6a363e514bff0d4ec954fc4034b: 2024-11-16T10:00:05,363 INFO [RS:0;037c58a19ddb:37267-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731751192181.697cd6a363e514bff0d4ec954fc4034b., storeName=697cd6a363e514bff0d4ec954fc4034b/A, priority=13, startTime=1731751205326; duration=0sec 2024-11-16T10:00:05,363 DEBUG [RS:0;037c58a19ddb:37267-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-16T10:00:05,363 DEBUG [RS:0;037c58a19ddb:37267-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 697cd6a363e514bff0d4ec954fc4034b:A 2024-11-16T10:00:05,367 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/697cd6a363e514bff0d4ec954fc4034b/.tmp/B/df81870aa751418da08296e961c67ac3 is 50, key is test_row_0/B:col10/1731751204930/Put/seqid=0 2024-11-16T10:00:05,371 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37591 is added to blk_1073742316_1492 (size=12151) 2024-11-16T10:00:05,371 INFO [RS:0;037c58a19ddb:37267-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 697cd6a363e514bff0d4ec954fc4034b#C#compaction#419 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-16T10:00:05,371 DEBUG [RS:0;037c58a19ddb:37267-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/697cd6a363e514bff0d4ec954fc4034b/.tmp/C/cb29188c3e284c5aafb352c461876405 is 50, key is test_row_0/C:col10/1731751203690/Put/seqid=0 2024-11-16T10:00:05,377 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=697cd6a363e514bff0d4ec954fc4034b, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T10:00:05,377 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] ipc.CallRunner(138): callId: 87 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36012 deadline: 1731751265360, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=697cd6a363e514bff0d4ec954fc4034b, server=037c58a19ddb,37267,1731751077844 2024-11-16T10:00:05,377 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=697cd6a363e514bff0d4ec954fc4034b, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T10:00:05,377 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] ipc.CallRunner(138): callId: 95 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36058 deadline: 1731751265364, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=697cd6a363e514bff0d4ec954fc4034b, server=037c58a19ddb,37267,1731751077844 2024-11-16T10:00:05,377 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731751192181.697cd6a363e514bff0d4ec954fc4034b., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 , the old value is region=TestAcidGuarantees,,1731751192181.697cd6a363e514bff0d4ec954fc4034b., hostname=037c58a19ddb,37267,1731751077844, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=697cd6a363e514bff0d4ec954fc4034b, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:00:05,377 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731751192181.697cd6a363e514bff0d4ec954fc4034b., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=697cd6a363e514bff0d4ec954fc4034b, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:00:05,377 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731751192181.697cd6a363e514bff0d4ec954fc4034b., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 because the exception is null or not the one we care about 2024-11-16T10:00:05,377 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731751192181.697cd6a363e514bff0d4ec954fc4034b., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 , the old value is region=TestAcidGuarantees,,1731751192181.697cd6a363e514bff0d4ec954fc4034b., hostname=037c58a19ddb,37267,1731751077844, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=697cd6a363e514bff0d4ec954fc4034b, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:00:05,377 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731751192181.697cd6a363e514bff0d4ec954fc4034b., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=697cd6a363e514bff0d4ec954fc4034b, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:00:05,378 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731751192181.697cd6a363e514bff0d4ec954fc4034b., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 because the exception is null or not the one we care about 2024-11-16T10:00:05,381 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37591 is added to blk_1073742317_1493 (size=12459) 2024-11-16T10:00:05,391 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=697cd6a363e514bff0d4ec954fc4034b, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T10:00:05,391 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36086 deadline: 1731751265377, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=697cd6a363e514bff0d4ec954fc4034b, server=037c58a19ddb,37267,1731751077844 2024-11-16T10:00:05,391 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=697cd6a363e514bff0d4ec954fc4034b, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T10:00:05,391 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] ipc.CallRunner(138): callId: 84 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36022 deadline: 1731751265377, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=697cd6a363e514bff0d4ec954fc4034b, server=037c58a19ddb,37267,1731751077844 2024-11-16T10:00:05,391 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731751192181.697cd6a363e514bff0d4ec954fc4034b., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 , the old value is region=TestAcidGuarantees,,1731751192181.697cd6a363e514bff0d4ec954fc4034b., hostname=037c58a19ddb,37267,1731751077844, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=697cd6a363e514bff0d4ec954fc4034b, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:00:05,392 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731751192181.697cd6a363e514bff0d4ec954fc4034b., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=697cd6a363e514bff0d4ec954fc4034b, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:00:05,392 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731751192181.697cd6a363e514bff0d4ec954fc4034b., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 because the exception is null or not the one we care about 2024-11-16T10:00:05,392 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731751192181.697cd6a363e514bff0d4ec954fc4034b., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 , the old value is region=TestAcidGuarantees,,1731751192181.697cd6a363e514bff0d4ec954fc4034b., hostname=037c58a19ddb,37267,1731751077844, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=697cd6a363e514bff0d4ec954fc4034b, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:00:05,392 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731751192181.697cd6a363e514bff0d4ec954fc4034b., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=697cd6a363e514bff0d4ec954fc4034b, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:00:05,392 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731751192181.697cd6a363e514bff0d4ec954fc4034b., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 because the exception is null or not the one we care about 2024-11-16T10:00:05,444 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=37267 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=128 2024-11-16T10:00:05,445 DEBUG [RS_FLUSH_OPERATIONS-regionserver/037c58a19ddb:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731751192181.697cd6a363e514bff0d4ec954fc4034b. 2024-11-16T10:00:05,445 DEBUG [RS_FLUSH_OPERATIONS-regionserver/037c58a19ddb:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731751192181.697cd6a363e514bff0d4ec954fc4034b. as already flushing 2024-11-16T10:00:05,445 DEBUG [RS_FLUSH_OPERATIONS-regionserver/037c58a19ddb:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731751192181.697cd6a363e514bff0d4ec954fc4034b. 2024-11-16T10:00:05,445 ERROR [RS_FLUSH_OPERATIONS-regionserver/037c58a19ddb:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] handler.RSProcedureHandler(58): pid=128 java.io.IOException: Unable to complete flush {ENCODED => 697cd6a363e514bff0d4ec954fc4034b, NAME => 'TestAcidGuarantees,,1731751192181.697cd6a363e514bff0d4ec954fc4034b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T10:00:05,445 DEBUG [RS_FLUSH_OPERATIONS-regionserver/037c58a19ddb:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=128 java.io.IOException: Unable to complete flush {ENCODED => 697cd6a363e514bff0d4ec954fc4034b, NAME => 'TestAcidGuarantees,,1731751192181.697cd6a363e514bff0d4ec954fc4034b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T10:00:05,446 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37877 {}] master.HMaster(4177): Remote procedure failed, pid=128 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 697cd6a363e514bff0d4ec954fc4034b, NAME => 'TestAcidGuarantees,,1731751192181.697cd6a363e514bff0d4ec954fc4034b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 697cd6a363e514bff0d4ec954fc4034b, NAME => 'TestAcidGuarantees,,1731751192181.697cd6a363e514bff0d4ec954fc4034b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T10:00:05,491 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=697cd6a363e514bff0d4ec954fc4034b, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T10:00:05,492 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] ipc.CallRunner(138): callId: 96 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36058 deadline: 1731751265483, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=697cd6a363e514bff0d4ec954fc4034b, server=037c58a19ddb,37267,1731751077844 2024-11-16T10:00:05,492 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=697cd6a363e514bff0d4ec954fc4034b, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T10:00:05,492 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] ipc.CallRunner(138): callId: 88 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36012 deadline: 1731751265483, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=697cd6a363e514bff0d4ec954fc4034b, server=037c58a19ddb,37267,1731751077844 2024-11-16T10:00:05,492 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731751192181.697cd6a363e514bff0d4ec954fc4034b., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 , the old value is region=TestAcidGuarantees,,1731751192181.697cd6a363e514bff0d4ec954fc4034b., hostname=037c58a19ddb,37267,1731751077844, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=697cd6a363e514bff0d4ec954fc4034b, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:00:05,492 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731751192181.697cd6a363e514bff0d4ec954fc4034b., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=697cd6a363e514bff0d4ec954fc4034b, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:00:05,492 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731751192181.697cd6a363e514bff0d4ec954fc4034b., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 because the exception is null or not the one we care about 2024-11-16T10:00:05,494 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731751192181.697cd6a363e514bff0d4ec954fc4034b., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 , the old value is region=TestAcidGuarantees,,1731751192181.697cd6a363e514bff0d4ec954fc4034b., hostname=037c58a19ddb,37267,1731751077844, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=697cd6a363e514bff0d4ec954fc4034b, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:00:05,494 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731751192181.697cd6a363e514bff0d4ec954fc4034b., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=697cd6a363e514bff0d4ec954fc4034b, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:00:05,494 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731751192181.697cd6a363e514bff0d4ec954fc4034b., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 because the exception is null or not the one we care about 2024-11-16T10:00:05,510 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=697cd6a363e514bff0d4ec954fc4034b, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T10:00:05,510 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] ipc.CallRunner(138): callId: 85 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36022 deadline: 1731751265503, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=697cd6a363e514bff0d4ec954fc4034b, server=037c58a19ddb,37267,1731751077844 2024-11-16T10:00:05,510 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=697cd6a363e514bff0d4ec954fc4034b, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T10:00:05,510 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731751192181.697cd6a363e514bff0d4ec954fc4034b., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 , the old value is region=TestAcidGuarantees,,1731751192181.697cd6a363e514bff0d4ec954fc4034b., hostname=037c58a19ddb,37267,1731751077844, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=697cd6a363e514bff0d4ec954fc4034b, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:00:05,510 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731751192181.697cd6a363e514bff0d4ec954fc4034b., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=697cd6a363e514bff0d4ec954fc4034b, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:00:05,510 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36086 deadline: 1731751265504, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=697cd6a363e514bff0d4ec954fc4034b, server=037c58a19ddb,37267,1731751077844 2024-11-16T10:00:05,510 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731751192181.697cd6a363e514bff0d4ec954fc4034b., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 because the exception is null or not the one we care about 2024-11-16T10:00:05,511 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731751192181.697cd6a363e514bff0d4ec954fc4034b., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 , the old value is region=TestAcidGuarantees,,1731751192181.697cd6a363e514bff0d4ec954fc4034b., hostname=037c58a19ddb,37267,1731751077844, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=697cd6a363e514bff0d4ec954fc4034b, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:00:05,511 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731751192181.697cd6a363e514bff0d4ec954fc4034b., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=697cd6a363e514bff0d4ec954fc4034b, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:00:05,511 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731751192181.697cd6a363e514bff0d4ec954fc4034b., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 because the exception is null or not the one we care about 2024-11-16T10:00:05,598 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=37267 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=128 2024-11-16T10:00:05,598 DEBUG [RS_FLUSH_OPERATIONS-regionserver/037c58a19ddb:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731751192181.697cd6a363e514bff0d4ec954fc4034b. 2024-11-16T10:00:05,598 DEBUG [RS_FLUSH_OPERATIONS-regionserver/037c58a19ddb:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731751192181.697cd6a363e514bff0d4ec954fc4034b. as already flushing 2024-11-16T10:00:05,598 DEBUG [RS_FLUSH_OPERATIONS-regionserver/037c58a19ddb:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731751192181.697cd6a363e514bff0d4ec954fc4034b. 2024-11-16T10:00:05,598 ERROR [RS_FLUSH_OPERATIONS-regionserver/037c58a19ddb:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] handler.RSProcedureHandler(58): pid=128 java.io.IOException: Unable to complete flush {ENCODED => 697cd6a363e514bff0d4ec954fc4034b, NAME => 'TestAcidGuarantees,,1731751192181.697cd6a363e514bff0d4ec954fc4034b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T10:00:05,598 DEBUG [RS_FLUSH_OPERATIONS-regionserver/037c58a19ddb:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=128 java.io.IOException: Unable to complete flush {ENCODED => 697cd6a363e514bff0d4ec954fc4034b, NAME => 'TestAcidGuarantees,,1731751192181.697cd6a363e514bff0d4ec954fc4034b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T10:00:05,599 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37877 {}] master.HMaster(4177): Remote procedure failed, pid=128 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 697cd6a363e514bff0d4ec954fc4034b, NAME => 'TestAcidGuarantees,,1731751192181.697cd6a363e514bff0d4ec954fc4034b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 697cd6a363e514bff0d4ec954fc4034b, NAME => 'TestAcidGuarantees,,1731751192181.697cd6a363e514bff0d4ec954fc4034b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T10:00:05,613 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37877 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=127 2024-11-16T10:00:05,708 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=697cd6a363e514bff0d4ec954fc4034b, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T10:00:05,708 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] ipc.CallRunner(138): callId: 97 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36058 deadline: 1731751265703, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=697cd6a363e514bff0d4ec954fc4034b, server=037c58a19ddb,37267,1731751077844 2024-11-16T10:00:05,709 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=697cd6a363e514bff0d4ec954fc4034b, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T10:00:05,709 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731751192181.697cd6a363e514bff0d4ec954fc4034b., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 , the old value is region=TestAcidGuarantees,,1731751192181.697cd6a363e514bff0d4ec954fc4034b., hostname=037c58a19ddb,37267,1731751077844, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=697cd6a363e514bff0d4ec954fc4034b, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:00:05,709 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731751192181.697cd6a363e514bff0d4ec954fc4034b., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=697cd6a363e514bff0d4ec954fc4034b, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:00:05,709 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] ipc.CallRunner(138): callId: 89 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36012 deadline: 1731751265703, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=697cd6a363e514bff0d4ec954fc4034b, server=037c58a19ddb,37267,1731751077844 2024-11-16T10:00:05,709 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731751192181.697cd6a363e514bff0d4ec954fc4034b., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 because the exception is null or not the one we care about 2024-11-16T10:00:05,709 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731751192181.697cd6a363e514bff0d4ec954fc4034b., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 , the old value is region=TestAcidGuarantees,,1731751192181.697cd6a363e514bff0d4ec954fc4034b., hostname=037c58a19ddb,37267,1731751077844, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=697cd6a363e514bff0d4ec954fc4034b, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:00:05,709 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731751192181.697cd6a363e514bff0d4ec954fc4034b., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=697cd6a363e514bff0d4ec954fc4034b, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:00:05,709 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731751192181.697cd6a363e514bff0d4ec954fc4034b., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 because the exception is null or not the one we care about 2024-11-16T10:00:05,720 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=697cd6a363e514bff0d4ec954fc4034b, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T10:00:05,720 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] ipc.CallRunner(138): callId: 86 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36022 deadline: 1731751265712, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=697cd6a363e514bff0d4ec954fc4034b, server=037c58a19ddb,37267,1731751077844 2024-11-16T10:00:05,720 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=697cd6a363e514bff0d4ec954fc4034b, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T10:00:05,720 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36086 deadline: 1731751265712, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=697cd6a363e514bff0d4ec954fc4034b, server=037c58a19ddb,37267,1731751077844 2024-11-16T10:00:05,720 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731751192181.697cd6a363e514bff0d4ec954fc4034b., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 , the old value is region=TestAcidGuarantees,,1731751192181.697cd6a363e514bff0d4ec954fc4034b., hostname=037c58a19ddb,37267,1731751077844, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=697cd6a363e514bff0d4ec954fc4034b, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:00:05,720 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731751192181.697cd6a363e514bff0d4ec954fc4034b., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=697cd6a363e514bff0d4ec954fc4034b, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:00:05,720 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731751192181.697cd6a363e514bff0d4ec954fc4034b., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 because the exception is null or not the one we care about 2024-11-16T10:00:05,720 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731751192181.697cd6a363e514bff0d4ec954fc4034b., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 , the old value is region=TestAcidGuarantees,,1731751192181.697cd6a363e514bff0d4ec954fc4034b., hostname=037c58a19ddb,37267,1731751077844, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=697cd6a363e514bff0d4ec954fc4034b, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:00:05,720 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731751192181.697cd6a363e514bff0d4ec954fc4034b., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=697cd6a363e514bff0d4ec954fc4034b, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:00:05,720 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731751192181.697cd6a363e514bff0d4ec954fc4034b., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 because the exception is null or not the one we care about 2024-11-16T10:00:05,750 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=37267 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=128 2024-11-16T10:00:05,751 DEBUG [RS_FLUSH_OPERATIONS-regionserver/037c58a19ddb:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731751192181.697cd6a363e514bff0d4ec954fc4034b. 2024-11-16T10:00:05,751 DEBUG [RS_FLUSH_OPERATIONS-regionserver/037c58a19ddb:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731751192181.697cd6a363e514bff0d4ec954fc4034b. as already flushing 2024-11-16T10:00:05,751 DEBUG [RS_FLUSH_OPERATIONS-regionserver/037c58a19ddb:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731751192181.697cd6a363e514bff0d4ec954fc4034b. 2024-11-16T10:00:05,751 ERROR [RS_FLUSH_OPERATIONS-regionserver/037c58a19ddb:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] handler.RSProcedureHandler(58): pid=128 java.io.IOException: Unable to complete flush {ENCODED => 697cd6a363e514bff0d4ec954fc4034b, NAME => 'TestAcidGuarantees,,1731751192181.697cd6a363e514bff0d4ec954fc4034b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T10:00:05,751 DEBUG [RS_FLUSH_OPERATIONS-regionserver/037c58a19ddb:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=128 java.io.IOException: Unable to complete flush {ENCODED => 697cd6a363e514bff0d4ec954fc4034b, NAME => 'TestAcidGuarantees,,1731751192181.697cd6a363e514bff0d4ec954fc4034b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T10:00:05,752 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37877 {}] master.HMaster(4177): Remote procedure failed, pid=128 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 697cd6a363e514bff0d4ec954fc4034b, NAME => 'TestAcidGuarantees,,1731751192181.697cd6a363e514bff0d4ec954fc4034b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 697cd6a363e514bff0d4ec954fc4034b, NAME => 'TestAcidGuarantees,,1731751192181.697cd6a363e514bff0d4ec954fc4034b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T10:00:05,771 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=233 (bloomFilter=true), to=hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/697cd6a363e514bff0d4ec954fc4034b/.tmp/B/df81870aa751418da08296e961c67ac3 2024-11-16T10:00:05,783 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/697cd6a363e514bff0d4ec954fc4034b/.tmp/C/1a1514cb66b8456bab35a7ce294b1663 is 50, key is test_row_0/C:col10/1731751204930/Put/seqid=0 2024-11-16T10:00:05,787 DEBUG [RS:0;037c58a19ddb:37267-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/697cd6a363e514bff0d4ec954fc4034b/.tmp/C/cb29188c3e284c5aafb352c461876405 as hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/697cd6a363e514bff0d4ec954fc4034b/C/cb29188c3e284c5aafb352c461876405 2024-11-16T10:00:05,792 INFO [RS:0;037c58a19ddb:37267-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 697cd6a363e514bff0d4ec954fc4034b/C of 697cd6a363e514bff0d4ec954fc4034b into cb29188c3e284c5aafb352c461876405(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-16T10:00:05,792 DEBUG [RS:0;037c58a19ddb:37267-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 697cd6a363e514bff0d4ec954fc4034b: 2024-11-16T10:00:05,792 INFO [RS:0;037c58a19ddb:37267-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731751192181.697cd6a363e514bff0d4ec954fc4034b., storeName=697cd6a363e514bff0d4ec954fc4034b/C, priority=13, startTime=1731751205327; duration=0sec 2024-11-16T10:00:05,792 DEBUG [RS:0;037c58a19ddb:37267-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-16T10:00:05,792 DEBUG [RS:0;037c58a19ddb:37267-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 697cd6a363e514bff0d4ec954fc4034b:C 2024-11-16T10:00:05,796 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37591 is added to blk_1073742318_1494 (size=12151) 2024-11-16T10:00:05,798 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=233 (bloomFilter=true), to=hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/697cd6a363e514bff0d4ec954fc4034b/.tmp/C/1a1514cb66b8456bab35a7ce294b1663 2024-11-16T10:00:05,801 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/697cd6a363e514bff0d4ec954fc4034b/.tmp/A/3cecc1d82dd343439dbccdd090ece5fd as hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/697cd6a363e514bff0d4ec954fc4034b/A/3cecc1d82dd343439dbccdd090ece5fd 2024-11-16T10:00:05,805 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/697cd6a363e514bff0d4ec954fc4034b/A/3cecc1d82dd343439dbccdd090ece5fd, entries=150, sequenceid=233, filesize=11.9 K 2024-11-16T10:00:05,806 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/697cd6a363e514bff0d4ec954fc4034b/.tmp/B/df81870aa751418da08296e961c67ac3 as hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/697cd6a363e514bff0d4ec954fc4034b/B/df81870aa751418da08296e961c67ac3 2024-11-16T10:00:05,811 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/697cd6a363e514bff0d4ec954fc4034b/B/df81870aa751418da08296e961c67ac3, entries=150, sequenceid=233, filesize=11.9 K 2024-11-16T10:00:05,812 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/697cd6a363e514bff0d4ec954fc4034b/.tmp/C/1a1514cb66b8456bab35a7ce294b1663 as hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/697cd6a363e514bff0d4ec954fc4034b/C/1a1514cb66b8456bab35a7ce294b1663 2024-11-16T10:00:05,815 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/697cd6a363e514bff0d4ec954fc4034b/C/1a1514cb66b8456bab35a7ce294b1663, entries=150, sequenceid=233, filesize=11.9 K 2024-11-16T10:00:05,816 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~147.60 KB/151140, heapSize ~387.42 KB/396720, currentSize=60.38 KB/61830 for 697cd6a363e514bff0d4ec954fc4034b in 482ms, sequenceid=233, compaction requested=false 2024-11-16T10:00:05,817 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 697cd6a363e514bff0d4ec954fc4034b: 2024-11-16T10:00:05,903 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=37267 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=128 2024-11-16T10:00:05,904 DEBUG [RS_FLUSH_OPERATIONS-regionserver/037c58a19ddb:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731751192181.697cd6a363e514bff0d4ec954fc4034b. 2024-11-16T10:00:05,904 INFO [RS_FLUSH_OPERATIONS-regionserver/037c58a19ddb:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HRegion(2902): Flushing 697cd6a363e514bff0d4ec954fc4034b 3/3 column families, dataSize=60.38 KB heapSize=158.95 KB 2024-11-16T10:00:05,908 DEBUG [RS_FLUSH_OPERATIONS-regionserver/037c58a19ddb:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/697cd6a363e514bff0d4ec954fc4034b/.tmp/A/5b4e2ff342e54bdc84037fe744487a35 is 50, key is test_row_0/A:col10/1731751205374/Put/seqid=0 2024-11-16T10:00:05,912 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37591 is added to blk_1073742319_1495 (size=12151) 2024-11-16T10:00:06,018 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] regionserver.HRegion(8855): Flush requested on 697cd6a363e514bff0d4ec954fc4034b 2024-11-16T10:00:06,018 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731751192181.697cd6a363e514bff0d4ec954fc4034b. as already flushing 2024-11-16T10:00:06,073 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=697cd6a363e514bff0d4ec954fc4034b, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T10:00:06,074 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] ipc.CallRunner(138): callId: 96 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36012 deadline: 1731751266063, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=697cd6a363e514bff0d4ec954fc4034b, server=037c58a19ddb,37267,1731751077844 2024-11-16T10:00:06,074 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731751192181.697cd6a363e514bff0d4ec954fc4034b., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 , the old value is region=TestAcidGuarantees,,1731751192181.697cd6a363e514bff0d4ec954fc4034b., hostname=037c58a19ddb,37267,1731751077844, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=697cd6a363e514bff0d4ec954fc4034b, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:00:06,074 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=697cd6a363e514bff0d4ec954fc4034b, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T10:00:06,074 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731751192181.697cd6a363e514bff0d4ec954fc4034b., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=697cd6a363e514bff0d4ec954fc4034b, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:00:06,074 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731751192181.697cd6a363e514bff0d4ec954fc4034b., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 because the exception is null or not the one we care about 2024-11-16T10:00:06,074 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] ipc.CallRunner(138): callId: 105 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36058 deadline: 1731751266064, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=697cd6a363e514bff0d4ec954fc4034b, server=037c58a19ddb,37267,1731751077844 2024-11-16T10:00:06,074 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731751192181.697cd6a363e514bff0d4ec954fc4034b., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 , the old value is region=TestAcidGuarantees,,1731751192181.697cd6a363e514bff0d4ec954fc4034b., hostname=037c58a19ddb,37267,1731751077844, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=697cd6a363e514bff0d4ec954fc4034b, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:00:06,074 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=697cd6a363e514bff0d4ec954fc4034b, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T10:00:06,074 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731751192181.697cd6a363e514bff0d4ec954fc4034b., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=697cd6a363e514bff0d4ec954fc4034b, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:00:06,075 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731751192181.697cd6a363e514bff0d4ec954fc4034b., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 because the exception is null or not the one we care about 2024-11-16T10:00:06,075 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] ipc.CallRunner(138): callId: 91 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36022 deadline: 1731751266065, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=697cd6a363e514bff0d4ec954fc4034b, server=037c58a19ddb,37267,1731751077844 2024-11-16T10:00:06,075 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=697cd6a363e514bff0d4ec954fc4034b, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T10:00:06,075 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36086 deadline: 1731751266074, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=697cd6a363e514bff0d4ec954fc4034b, server=037c58a19ddb,37267,1731751077844 2024-11-16T10:00:06,075 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731751192181.697cd6a363e514bff0d4ec954fc4034b., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 , the old value is region=TestAcidGuarantees,,1731751192181.697cd6a363e514bff0d4ec954fc4034b., hostname=037c58a19ddb,37267,1731751077844, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=697cd6a363e514bff0d4ec954fc4034b, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:00:06,075 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731751192181.697cd6a363e514bff0d4ec954fc4034b., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 , the old value is region=TestAcidGuarantees,,1731751192181.697cd6a363e514bff0d4ec954fc4034b., hostname=037c58a19ddb,37267,1731751077844, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=697cd6a363e514bff0d4ec954fc4034b, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:00:06,075 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731751192181.697cd6a363e514bff0d4ec954fc4034b., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=697cd6a363e514bff0d4ec954fc4034b, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:00:06,075 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731751192181.697cd6a363e514bff0d4ec954fc4034b., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=697cd6a363e514bff0d4ec954fc4034b, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:00:06,075 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731751192181.697cd6a363e514bff0d4ec954fc4034b., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 because the exception is null or not the one we care about 2024-11-16T10:00:06,075 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731751192181.697cd6a363e514bff0d4ec954fc4034b., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 because the exception is null or not the one we care about 2024-11-16T10:00:06,123 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37877 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=127 2024-11-16T10:00:06,190 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=697cd6a363e514bff0d4ec954fc4034b, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T10:00:06,190 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] ipc.CallRunner(138): callId: 97 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36012 deadline: 1731751266183, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=697cd6a363e514bff0d4ec954fc4034b, server=037c58a19ddb,37267,1731751077844 2024-11-16T10:00:06,190 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=697cd6a363e514bff0d4ec954fc4034b, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T10:00:06,190 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731751192181.697cd6a363e514bff0d4ec954fc4034b., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 , the old value is region=TestAcidGuarantees,,1731751192181.697cd6a363e514bff0d4ec954fc4034b., hostname=037c58a19ddb,37267,1731751077844, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=697cd6a363e514bff0d4ec954fc4034b, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:00:06,190 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731751192181.697cd6a363e514bff0d4ec954fc4034b., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=697cd6a363e514bff0d4ec954fc4034b, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:00:06,190 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731751192181.697cd6a363e514bff0d4ec954fc4034b., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 because the exception is null or not the one we care about 2024-11-16T10:00:06,190 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] ipc.CallRunner(138): callId: 106 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36058 deadline: 1731751266183, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=697cd6a363e514bff0d4ec954fc4034b, server=037c58a19ddb,37267,1731751077844 2024-11-16T10:00:06,190 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=697cd6a363e514bff0d4ec954fc4034b, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T10:00:06,190 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731751192181.697cd6a363e514bff0d4ec954fc4034b., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 , the old value is region=TestAcidGuarantees,,1731751192181.697cd6a363e514bff0d4ec954fc4034b., hostname=037c58a19ddb,37267,1731751077844, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=697cd6a363e514bff0d4ec954fc4034b, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:00:06,190 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731751192181.697cd6a363e514bff0d4ec954fc4034b., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=697cd6a363e514bff0d4ec954fc4034b, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:00:06,191 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731751192181.697cd6a363e514bff0d4ec954fc4034b., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 because the exception is null or not the one we care about 2024-11-16T10:00:06,191 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] ipc.CallRunner(138): callId: 92 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36022 deadline: 1731751266183, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=697cd6a363e514bff0d4ec954fc4034b, server=037c58a19ddb,37267,1731751077844 2024-11-16T10:00:06,191 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731751192181.697cd6a363e514bff0d4ec954fc4034b., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 , the old value is region=TestAcidGuarantees,,1731751192181.697cd6a363e514bff0d4ec954fc4034b., hostname=037c58a19ddb,37267,1731751077844, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=697cd6a363e514bff0d4ec954fc4034b, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:00:06,191 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=697cd6a363e514bff0d4ec954fc4034b, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T10:00:06,191 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731751192181.697cd6a363e514bff0d4ec954fc4034b., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=697cd6a363e514bff0d4ec954fc4034b, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:00:06,191 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731751192181.697cd6a363e514bff0d4ec954fc4034b., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 because the exception is null or not the one we care about 2024-11-16T10:00:06,191 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36086 deadline: 1731751266183, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=697cd6a363e514bff0d4ec954fc4034b, server=037c58a19ddb,37267,1731751077844 2024-11-16T10:00:06,191 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731751192181.697cd6a363e514bff0d4ec954fc4034b., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 , the old value is region=TestAcidGuarantees,,1731751192181.697cd6a363e514bff0d4ec954fc4034b., hostname=037c58a19ddb,37267,1731751077844, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=697cd6a363e514bff0d4ec954fc4034b, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:00:06,191 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731751192181.697cd6a363e514bff0d4ec954fc4034b., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=697cd6a363e514bff0d4ec954fc4034b, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:00:06,191 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731751192181.697cd6a363e514bff0d4ec954fc4034b., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 because the exception is null or not the one we care about 2024-11-16T10:00:06,313 INFO [RS_FLUSH_OPERATIONS-regionserver/037c58a19ddb:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=248 (bloomFilter=true), to=hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/697cd6a363e514bff0d4ec954fc4034b/.tmp/A/5b4e2ff342e54bdc84037fe744487a35 2024-11-16T10:00:06,320 DEBUG [RS_FLUSH_OPERATIONS-regionserver/037c58a19ddb:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/697cd6a363e514bff0d4ec954fc4034b/.tmp/B/8048b8cfb710419bb4932c2b21933a5c is 50, key is test_row_0/B:col10/1731751205374/Put/seqid=0 2024-11-16T10:00:06,323 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37591 is added to blk_1073742320_1496 (size=12151) 2024-11-16T10:00:06,324 INFO [RS_FLUSH_OPERATIONS-regionserver/037c58a19ddb:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=248 (bloomFilter=true), to=hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/697cd6a363e514bff0d4ec954fc4034b/.tmp/B/8048b8cfb710419bb4932c2b21933a5c 2024-11-16T10:00:06,331 DEBUG [RS_FLUSH_OPERATIONS-regionserver/037c58a19ddb:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/697cd6a363e514bff0d4ec954fc4034b/.tmp/C/699da832fda74291b53ee546204927b7 is 50, key is test_row_0/C:col10/1731751205374/Put/seqid=0 2024-11-16T10:00:06,335 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37591 is added to blk_1073742321_1497 (size=12151) 2024-11-16T10:00:06,401 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=697cd6a363e514bff0d4ec954fc4034b, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T10:00:06,401 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] ipc.CallRunner(138): callId: 98 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36012 deadline: 1731751266393, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=697cd6a363e514bff0d4ec954fc4034b, server=037c58a19ddb,37267,1731751077844 2024-11-16T10:00:06,401 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=697cd6a363e514bff0d4ec954fc4034b, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T10:00:06,401 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731751192181.697cd6a363e514bff0d4ec954fc4034b., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 , the old value is region=TestAcidGuarantees,,1731751192181.697cd6a363e514bff0d4ec954fc4034b., hostname=037c58a19ddb,37267,1731751077844, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=697cd6a363e514bff0d4ec954fc4034b, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:00:06,401 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731751192181.697cd6a363e514bff0d4ec954fc4034b., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=697cd6a363e514bff0d4ec954fc4034b, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:00:06,401 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] ipc.CallRunner(138): callId: 107 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36058 deadline: 1731751266393, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=697cd6a363e514bff0d4ec954fc4034b, server=037c58a19ddb,37267,1731751077844 2024-11-16T10:00:06,401 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731751192181.697cd6a363e514bff0d4ec954fc4034b., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 because the exception is null or not the one we care about 2024-11-16T10:00:06,402 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731751192181.697cd6a363e514bff0d4ec954fc4034b., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 , the old value is region=TestAcidGuarantees,,1731751192181.697cd6a363e514bff0d4ec954fc4034b., hostname=037c58a19ddb,37267,1731751077844, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=697cd6a363e514bff0d4ec954fc4034b, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:00:06,402 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731751192181.697cd6a363e514bff0d4ec954fc4034b., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=697cd6a363e514bff0d4ec954fc4034b, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:00:06,402 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731751192181.697cd6a363e514bff0d4ec954fc4034b., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 because the exception is null or not the one we care about 2024-11-16T10:00:06,412 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=697cd6a363e514bff0d4ec954fc4034b, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T10:00:06,412 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] ipc.CallRunner(138): callId: 93 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36022 deadline: 1731751266403, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=697cd6a363e514bff0d4ec954fc4034b, server=037c58a19ddb,37267,1731751077844 2024-11-16T10:00:06,412 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=697cd6a363e514bff0d4ec954fc4034b, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T10:00:06,412 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36086 deadline: 1731751266403, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=697cd6a363e514bff0d4ec954fc4034b, server=037c58a19ddb,37267,1731751077844 2024-11-16T10:00:06,413 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731751192181.697cd6a363e514bff0d4ec954fc4034b., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 , the old value is region=TestAcidGuarantees,,1731751192181.697cd6a363e514bff0d4ec954fc4034b., hostname=037c58a19ddb,37267,1731751077844, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=697cd6a363e514bff0d4ec954fc4034b, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:00:06,413 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731751192181.697cd6a363e514bff0d4ec954fc4034b., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=697cd6a363e514bff0d4ec954fc4034b, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:00:06,413 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731751192181.697cd6a363e514bff0d4ec954fc4034b., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 because the exception is null or not the one we care about 2024-11-16T10:00:06,413 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731751192181.697cd6a363e514bff0d4ec954fc4034b., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 , the old value is region=TestAcidGuarantees,,1731751192181.697cd6a363e514bff0d4ec954fc4034b., hostname=037c58a19ddb,37267,1731751077844, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=697cd6a363e514bff0d4ec954fc4034b, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:00:06,413 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731751192181.697cd6a363e514bff0d4ec954fc4034b., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=697cd6a363e514bff0d4ec954fc4034b, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:00:06,413 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731751192181.697cd6a363e514bff0d4ec954fc4034b., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 because the exception is null or not the one we care about 2024-11-16T10:00:06,716 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=697cd6a363e514bff0d4ec954fc4034b, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T10:00:06,716 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] ipc.CallRunner(138): callId: 99 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36012 deadline: 1731751266712, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=697cd6a363e514bff0d4ec954fc4034b, server=037c58a19ddb,37267,1731751077844 2024-11-16T10:00:06,716 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=697cd6a363e514bff0d4ec954fc4034b, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T10:00:06,716 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731751192181.697cd6a363e514bff0d4ec954fc4034b., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 , the old value is region=TestAcidGuarantees,,1731751192181.697cd6a363e514bff0d4ec954fc4034b., hostname=037c58a19ddb,37267,1731751077844, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=697cd6a363e514bff0d4ec954fc4034b, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:00:06,716 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] ipc.CallRunner(138): callId: 108 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36058 deadline: 1731751266713, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=697cd6a363e514bff0d4ec954fc4034b, server=037c58a19ddb,37267,1731751077844 2024-11-16T10:00:06,716 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731751192181.697cd6a363e514bff0d4ec954fc4034b., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=697cd6a363e514bff0d4ec954fc4034b, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:00:06,716 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731751192181.697cd6a363e514bff0d4ec954fc4034b., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 because the exception is null or not the one we care about 2024-11-16T10:00:06,717 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731751192181.697cd6a363e514bff0d4ec954fc4034b., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 , the old value is region=TestAcidGuarantees,,1731751192181.697cd6a363e514bff0d4ec954fc4034b., hostname=037c58a19ddb,37267,1731751077844, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=697cd6a363e514bff0d4ec954fc4034b, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:00:06,717 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731751192181.697cd6a363e514bff0d4ec954fc4034b., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=697cd6a363e514bff0d4ec954fc4034b, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:00:06,717 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731751192181.697cd6a363e514bff0d4ec954fc4034b., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 because the exception is null or not the one we care about 2024-11-16T10:00:06,727 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=697cd6a363e514bff0d4ec954fc4034b, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T10:00:06,727 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] ipc.CallRunner(138): callId: 94 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36022 deadline: 1731751266723, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=697cd6a363e514bff0d4ec954fc4034b, server=037c58a19ddb,37267,1731751077844 2024-11-16T10:00:06,727 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=697cd6a363e514bff0d4ec954fc4034b, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T10:00:06,727 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36086 deadline: 1731751266723, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=697cd6a363e514bff0d4ec954fc4034b, server=037c58a19ddb,37267,1731751077844 2024-11-16T10:00:06,727 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731751192181.697cd6a363e514bff0d4ec954fc4034b., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 , the old value is region=TestAcidGuarantees,,1731751192181.697cd6a363e514bff0d4ec954fc4034b., hostname=037c58a19ddb,37267,1731751077844, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=697cd6a363e514bff0d4ec954fc4034b, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:00:06,727 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731751192181.697cd6a363e514bff0d4ec954fc4034b., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=697cd6a363e514bff0d4ec954fc4034b, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:00:06,727 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731751192181.697cd6a363e514bff0d4ec954fc4034b., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 because the exception is null or not the one we care about 2024-11-16T10:00:06,728 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731751192181.697cd6a363e514bff0d4ec954fc4034b., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 , the old value is region=TestAcidGuarantees,,1731751192181.697cd6a363e514bff0d4ec954fc4034b., hostname=037c58a19ddb,37267,1731751077844, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=697cd6a363e514bff0d4ec954fc4034b, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:00:06,728 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731751192181.697cd6a363e514bff0d4ec954fc4034b., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=697cd6a363e514bff0d4ec954fc4034b, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:00:06,728 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731751192181.697cd6a363e514bff0d4ec954fc4034b., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 because the exception is null or not the one we care about 2024-11-16T10:00:06,735 INFO [RS_FLUSH_OPERATIONS-regionserver/037c58a19ddb:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=248 (bloomFilter=true), to=hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/697cd6a363e514bff0d4ec954fc4034b/.tmp/C/699da832fda74291b53ee546204927b7 2024-11-16T10:00:06,740 DEBUG [RS_FLUSH_OPERATIONS-regionserver/037c58a19ddb:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/697cd6a363e514bff0d4ec954fc4034b/.tmp/A/5b4e2ff342e54bdc84037fe744487a35 as hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/697cd6a363e514bff0d4ec954fc4034b/A/5b4e2ff342e54bdc84037fe744487a35 2024-11-16T10:00:06,743 INFO [RS_FLUSH_OPERATIONS-regionserver/037c58a19ddb:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/697cd6a363e514bff0d4ec954fc4034b/A/5b4e2ff342e54bdc84037fe744487a35, entries=150, sequenceid=248, filesize=11.9 K 2024-11-16T10:00:06,744 DEBUG [RS_FLUSH_OPERATIONS-regionserver/037c58a19ddb:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/697cd6a363e514bff0d4ec954fc4034b/.tmp/B/8048b8cfb710419bb4932c2b21933a5c as hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/697cd6a363e514bff0d4ec954fc4034b/B/8048b8cfb710419bb4932c2b21933a5c 2024-11-16T10:00:06,747 INFO [RS_FLUSH_OPERATIONS-regionserver/037c58a19ddb:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/697cd6a363e514bff0d4ec954fc4034b/B/8048b8cfb710419bb4932c2b21933a5c, entries=150, sequenceid=248, filesize=11.9 K 2024-11-16T10:00:06,748 DEBUG [RS_FLUSH_OPERATIONS-regionserver/037c58a19ddb:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/697cd6a363e514bff0d4ec954fc4034b/.tmp/C/699da832fda74291b53ee546204927b7 as hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/697cd6a363e514bff0d4ec954fc4034b/C/699da832fda74291b53ee546204927b7 2024-11-16T10:00:06,752 INFO [RS_FLUSH_OPERATIONS-regionserver/037c58a19ddb:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/697cd6a363e514bff0d4ec954fc4034b/C/699da832fda74291b53ee546204927b7, entries=150, sequenceid=248, filesize=11.9 K 2024-11-16T10:00:06,753 INFO [RS_FLUSH_OPERATIONS-regionserver/037c58a19ddb:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~158.91 KB/162720, currentSize=140.89 KB/144270 for 697cd6a363e514bff0d4ec954fc4034b in 849ms, sequenceid=248, compaction requested=true 2024-11-16T10:00:06,753 DEBUG [RS_FLUSH_OPERATIONS-regionserver/037c58a19ddb:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HRegion(2603): Flush status journal for 697cd6a363e514bff0d4ec954fc4034b: 2024-11-16T10:00:06,753 DEBUG [RS_FLUSH_OPERATIONS-regionserver/037c58a19ddb:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731751192181.697cd6a363e514bff0d4ec954fc4034b. 2024-11-16T10:00:06,753 DEBUG [RS_FLUSH_OPERATIONS-regionserver/037c58a19ddb:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=128 2024-11-16T10:00:06,753 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37877 {}] master.HMaster(4169): Remote procedure done, pid=128 2024-11-16T10:00:06,759 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=128, resume processing ppid=127 2024-11-16T10:00:06,759 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=128, ppid=127, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.7720 sec 2024-11-16T10:00:06,761 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=127, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=127, table=TestAcidGuarantees in 1.7770 sec 2024-11-16T10:00:07,132 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37877 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=127 2024-11-16T10:00:07,133 INFO [RPCClient-NioEventLoopGroup-4-10 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-16T10:00:07,133 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37877 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-16T10:00:07,134 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37877 {}] procedure2.ProcedureExecutor(1139): Stored pid=129, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=129, table=TestAcidGuarantees 2024-11-16T10:00:07,135 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37877 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=129 2024-11-16T10:00:07,135 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=129, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=129, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-16T10:00:07,136 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=129, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=129, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-16T10:00:07,136 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=130, ppid=129, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-16T10:00:07,230 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] regionserver.HRegion(8855): Flush requested on 697cd6a363e514bff0d4ec954fc4034b 2024-11-16T10:00:07,230 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-16T10:00:07,230 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 697cd6a363e514bff0d4ec954fc4034b 3/3 column families, dataSize=147.60 KB heapSize=387.47 KB 2024-11-16T10:00:07,239 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/697cd6a363e514bff0d4ec954fc4034b/.tmp/A/86ab58c9d50f47258676a412736ef737 is 50, key is test_row_0/A:col10/1731751206062/Put/seqid=0 2024-11-16T10:00:07,243 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37877 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=129 2024-11-16T10:00:07,245 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37591 is added to blk_1073742322_1498 (size=14741) 2024-11-16T10:00:07,288 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=37267 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=130 2024-11-16T10:00:07,288 DEBUG [RS_FLUSH_OPERATIONS-regionserver/037c58a19ddb:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731751192181.697cd6a363e514bff0d4ec954fc4034b. 2024-11-16T10:00:07,289 DEBUG [RS_FLUSH_OPERATIONS-regionserver/037c58a19ddb:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731751192181.697cd6a363e514bff0d4ec954fc4034b. as already flushing 2024-11-16T10:00:07,289 DEBUG [RS_FLUSH_OPERATIONS-regionserver/037c58a19ddb:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731751192181.697cd6a363e514bff0d4ec954fc4034b. 2024-11-16T10:00:07,289 ERROR [RS_FLUSH_OPERATIONS-regionserver/037c58a19ddb:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] handler.RSProcedureHandler(58): pid=130 java.io.IOException: Unable to complete flush {ENCODED => 697cd6a363e514bff0d4ec954fc4034b, NAME => 'TestAcidGuarantees,,1731751192181.697cd6a363e514bff0d4ec954fc4034b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T10:00:07,289 DEBUG [RS_FLUSH_OPERATIONS-regionserver/037c58a19ddb:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=130 java.io.IOException: Unable to complete flush {ENCODED => 697cd6a363e514bff0d4ec954fc4034b, NAME => 'TestAcidGuarantees,,1731751192181.697cd6a363e514bff0d4ec954fc4034b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T10:00:07,289 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37877 {}] master.HMaster(4177): Remote procedure failed, pid=130 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 697cd6a363e514bff0d4ec954fc4034b, NAME => 'TestAcidGuarantees,,1731751192181.697cd6a363e514bff0d4ec954fc4034b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 697cd6a363e514bff0d4ec954fc4034b, NAME => 'TestAcidGuarantees,,1731751192181.697cd6a363e514bff0d4ec954fc4034b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T10:00:07,293 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=697cd6a363e514bff0d4ec954fc4034b, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T10:00:07,294 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36086 deadline: 1731751267278, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=697cd6a363e514bff0d4ec954fc4034b, server=037c58a19ddb,37267,1731751077844 2024-11-16T10:00:07,294 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731751192181.697cd6a363e514bff0d4ec954fc4034b., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 , the old value is region=TestAcidGuarantees,,1731751192181.697cd6a363e514bff0d4ec954fc4034b., hostname=037c58a19ddb,37267,1731751077844, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=697cd6a363e514bff0d4ec954fc4034b, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:00:07,294 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731751192181.697cd6a363e514bff0d4ec954fc4034b., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=697cd6a363e514bff0d4ec954fc4034b, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:00:07,294 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731751192181.697cd6a363e514bff0d4ec954fc4034b., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 because the exception is null or not the one we care about 2024-11-16T10:00:07,304 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=697cd6a363e514bff0d4ec954fc4034b, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T10:00:07,304 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] ipc.CallRunner(138): callId: 103 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36012 deadline: 1731751267292, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=697cd6a363e514bff0d4ec954fc4034b, server=037c58a19ddb,37267,1731751077844 2024-11-16T10:00:07,304 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731751192181.697cd6a363e514bff0d4ec954fc4034b., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 , the old value is region=TestAcidGuarantees,,1731751192181.697cd6a363e514bff0d4ec954fc4034b., hostname=037c58a19ddb,37267,1731751077844, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=697cd6a363e514bff0d4ec954fc4034b, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:00:07,304 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=697cd6a363e514bff0d4ec954fc4034b, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T10:00:07,304 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731751192181.697cd6a363e514bff0d4ec954fc4034b., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=697cd6a363e514bff0d4ec954fc4034b, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:00:07,305 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731751192181.697cd6a363e514bff0d4ec954fc4034b., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 because the exception is null or not the one we care about 2024-11-16T10:00:07,305 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] ipc.CallRunner(138): callId: 112 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36058 deadline: 1731751267293, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=697cd6a363e514bff0d4ec954fc4034b, server=037c58a19ddb,37267,1731751077844 2024-11-16T10:00:07,305 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=697cd6a363e514bff0d4ec954fc4034b, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T10:00:07,305 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731751192181.697cd6a363e514bff0d4ec954fc4034b., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 , the old value is region=TestAcidGuarantees,,1731751192181.697cd6a363e514bff0d4ec954fc4034b., hostname=037c58a19ddb,37267,1731751077844, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=697cd6a363e514bff0d4ec954fc4034b, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:00:07,305 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] ipc.CallRunner(138): callId: 97 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36022 deadline: 1731751267294, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=697cd6a363e514bff0d4ec954fc4034b, server=037c58a19ddb,37267,1731751077844 2024-11-16T10:00:07,305 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731751192181.697cd6a363e514bff0d4ec954fc4034b., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=697cd6a363e514bff0d4ec954fc4034b, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:00:07,305 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731751192181.697cd6a363e514bff0d4ec954fc4034b., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 because the exception is null or not the one we care about 2024-11-16T10:00:07,305 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731751192181.697cd6a363e514bff0d4ec954fc4034b., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 , the old value is region=TestAcidGuarantees,,1731751192181.697cd6a363e514bff0d4ec954fc4034b., hostname=037c58a19ddb,37267,1731751077844, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=697cd6a363e514bff0d4ec954fc4034b, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:00:07,305 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731751192181.697cd6a363e514bff0d4ec954fc4034b., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=697cd6a363e514bff0d4ec954fc4034b, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:00:07,305 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731751192181.697cd6a363e514bff0d4ec954fc4034b., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 because the exception is null or not the one we care about 2024-11-16T10:00:07,411 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=697cd6a363e514bff0d4ec954fc4034b, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T10:00:07,411 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36086 deadline: 1731751267403, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=697cd6a363e514bff0d4ec954fc4034b, server=037c58a19ddb,37267,1731751077844 2024-11-16T10:00:07,412 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731751192181.697cd6a363e514bff0d4ec954fc4034b., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 , the old value is region=TestAcidGuarantees,,1731751192181.697cd6a363e514bff0d4ec954fc4034b., hostname=037c58a19ddb,37267,1731751077844, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=697cd6a363e514bff0d4ec954fc4034b, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:00:07,412 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731751192181.697cd6a363e514bff0d4ec954fc4034b., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=697cd6a363e514bff0d4ec954fc4034b, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:00:07,412 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731751192181.697cd6a363e514bff0d4ec954fc4034b., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 because the exception is null or not the one we care about 2024-11-16T10:00:07,421 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=697cd6a363e514bff0d4ec954fc4034b, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T10:00:07,421 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] ipc.CallRunner(138): callId: 104 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36012 deadline: 1731751267412, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=697cd6a363e514bff0d4ec954fc4034b, server=037c58a19ddb,37267,1731751077844 2024-11-16T10:00:07,422 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731751192181.697cd6a363e514bff0d4ec954fc4034b., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 , the old value is region=TestAcidGuarantees,,1731751192181.697cd6a363e514bff0d4ec954fc4034b., hostname=037c58a19ddb,37267,1731751077844, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=697cd6a363e514bff0d4ec954fc4034b, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:00:07,422 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731751192181.697cd6a363e514bff0d4ec954fc4034b., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=697cd6a363e514bff0d4ec954fc4034b, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:00:07,422 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=697cd6a363e514bff0d4ec954fc4034b, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T10:00:07,422 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731751192181.697cd6a363e514bff0d4ec954fc4034b., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 because the exception is null or not the one we care about 2024-11-16T10:00:07,422 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] ipc.CallRunner(138): callId: 98 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36022 deadline: 1731751267413, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=697cd6a363e514bff0d4ec954fc4034b, server=037c58a19ddb,37267,1731751077844 2024-11-16T10:00:07,422 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=697cd6a363e514bff0d4ec954fc4034b, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T10:00:07,422 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731751192181.697cd6a363e514bff0d4ec954fc4034b., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 , the old value is region=TestAcidGuarantees,,1731751192181.697cd6a363e514bff0d4ec954fc4034b., hostname=037c58a19ddb,37267,1731751077844, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=697cd6a363e514bff0d4ec954fc4034b, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:00:07,422 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] ipc.CallRunner(138): callId: 113 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36058 deadline: 1731751267413, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=697cd6a363e514bff0d4ec954fc4034b, server=037c58a19ddb,37267,1731751077844 2024-11-16T10:00:07,422 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731751192181.697cd6a363e514bff0d4ec954fc4034b., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=697cd6a363e514bff0d4ec954fc4034b, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:00:07,422 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731751192181.697cd6a363e514bff0d4ec954fc4034b., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 because the exception is null or not the one we care about 2024-11-16T10:00:07,422 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731751192181.697cd6a363e514bff0d4ec954fc4034b., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 , the old value is region=TestAcidGuarantees,,1731751192181.697cd6a363e514bff0d4ec954fc4034b., hostname=037c58a19ddb,37267,1731751077844, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=697cd6a363e514bff0d4ec954fc4034b, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:00:07,422 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731751192181.697cd6a363e514bff0d4ec954fc4034b., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=697cd6a363e514bff0d4ec954fc4034b, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:00:07,422 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731751192181.697cd6a363e514bff0d4ec954fc4034b., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 because the exception is null or not the one we care about 2024-11-16T10:00:07,441 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=37267 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=130 2024-11-16T10:00:07,442 DEBUG [RS_FLUSH_OPERATIONS-regionserver/037c58a19ddb:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731751192181.697cd6a363e514bff0d4ec954fc4034b. 2024-11-16T10:00:07,442 DEBUG [RS_FLUSH_OPERATIONS-regionserver/037c58a19ddb:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731751192181.697cd6a363e514bff0d4ec954fc4034b. as already flushing 2024-11-16T10:00:07,442 DEBUG [RS_FLUSH_OPERATIONS-regionserver/037c58a19ddb:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731751192181.697cd6a363e514bff0d4ec954fc4034b. 2024-11-16T10:00:07,442 ERROR [RS_FLUSH_OPERATIONS-regionserver/037c58a19ddb:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] handler.RSProcedureHandler(58): pid=130 java.io.IOException: Unable to complete flush {ENCODED => 697cd6a363e514bff0d4ec954fc4034b, NAME => 'TestAcidGuarantees,,1731751192181.697cd6a363e514bff0d4ec954fc4034b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T10:00:07,442 DEBUG [RS_FLUSH_OPERATIONS-regionserver/037c58a19ddb:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=130 java.io.IOException: Unable to complete flush {ENCODED => 697cd6a363e514bff0d4ec954fc4034b, NAME => 'TestAcidGuarantees,,1731751192181.697cd6a363e514bff0d4ec954fc4034b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T10:00:07,443 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37877 {}] master.HMaster(4177): Remote procedure failed, pid=130 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 697cd6a363e514bff0d4ec954fc4034b, NAME => 'TestAcidGuarantees,,1731751192181.697cd6a363e514bff0d4ec954fc4034b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 697cd6a363e514bff0d4ec954fc4034b, NAME => 'TestAcidGuarantees,,1731751192181.697cd6a363e514bff0d4ec954fc4034b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T10:00:07,453 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37877 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=129 2024-11-16T10:00:07,594 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=37267 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=130 2024-11-16T10:00:07,595 DEBUG [RS_FLUSH_OPERATIONS-regionserver/037c58a19ddb:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731751192181.697cd6a363e514bff0d4ec954fc4034b. 2024-11-16T10:00:07,595 DEBUG [RS_FLUSH_OPERATIONS-regionserver/037c58a19ddb:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731751192181.697cd6a363e514bff0d4ec954fc4034b. as already flushing 2024-11-16T10:00:07,595 DEBUG [RS_FLUSH_OPERATIONS-regionserver/037c58a19ddb:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731751192181.697cd6a363e514bff0d4ec954fc4034b. 2024-11-16T10:00:07,595 ERROR [RS_FLUSH_OPERATIONS-regionserver/037c58a19ddb:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] handler.RSProcedureHandler(58): pid=130 java.io.IOException: Unable to complete flush {ENCODED => 697cd6a363e514bff0d4ec954fc4034b, NAME => 'TestAcidGuarantees,,1731751192181.697cd6a363e514bff0d4ec954fc4034b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T10:00:07,595 DEBUG [RS_FLUSH_OPERATIONS-regionserver/037c58a19ddb:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=130 java.io.IOException: Unable to complete flush {ENCODED => 697cd6a363e514bff0d4ec954fc4034b, NAME => 'TestAcidGuarantees,,1731751192181.697cd6a363e514bff0d4ec954fc4034b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T10:00:07,596 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37877 {}] master.HMaster(4177): Remote procedure failed, pid=130 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 697cd6a363e514bff0d4ec954fc4034b, NAME => 'TestAcidGuarantees,,1731751192181.697cd6a363e514bff0d4ec954fc4034b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 697cd6a363e514bff0d4ec954fc4034b, NAME => 'TestAcidGuarantees,,1731751192181.697cd6a363e514bff0d4ec954fc4034b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T10:00:07,629 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=697cd6a363e514bff0d4ec954fc4034b, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T10:00:07,629 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36086 deadline: 1731751267622, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=697cd6a363e514bff0d4ec954fc4034b, server=037c58a19ddb,37267,1731751077844 2024-11-16T10:00:07,630 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731751192181.697cd6a363e514bff0d4ec954fc4034b., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 , the old value is region=TestAcidGuarantees,,1731751192181.697cd6a363e514bff0d4ec954fc4034b., hostname=037c58a19ddb,37267,1731751077844, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=697cd6a363e514bff0d4ec954fc4034b, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:00:07,630 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731751192181.697cd6a363e514bff0d4ec954fc4034b., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=697cd6a363e514bff0d4ec954fc4034b, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:00:07,630 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731751192181.697cd6a363e514bff0d4ec954fc4034b., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 because the exception is null or not the one we care about 2024-11-16T10:00:07,640 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=697cd6a363e514bff0d4ec954fc4034b, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T10:00:07,640 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] ipc.CallRunner(138): callId: 105 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36012 deadline: 1731751267633, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=697cd6a363e514bff0d4ec954fc4034b, server=037c58a19ddb,37267,1731751077844 2024-11-16T10:00:07,641 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=697cd6a363e514bff0d4ec954fc4034b, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T10:00:07,641 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731751192181.697cd6a363e514bff0d4ec954fc4034b., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 , the old value is region=TestAcidGuarantees,,1731751192181.697cd6a363e514bff0d4ec954fc4034b., hostname=037c58a19ddb,37267,1731751077844, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=697cd6a363e514bff0d4ec954fc4034b, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:00:07,641 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731751192181.697cd6a363e514bff0d4ec954fc4034b., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=697cd6a363e514bff0d4ec954fc4034b, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:00:07,641 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] ipc.CallRunner(138): callId: 99 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36022 deadline: 1731751267633, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=697cd6a363e514bff0d4ec954fc4034b, server=037c58a19ddb,37267,1731751077844 2024-11-16T10:00:07,641 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731751192181.697cd6a363e514bff0d4ec954fc4034b., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 because the exception is null or not the one we care about 2024-11-16T10:00:07,641 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=697cd6a363e514bff0d4ec954fc4034b, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T10:00:07,641 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731751192181.697cd6a363e514bff0d4ec954fc4034b., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 , the old value is region=TestAcidGuarantees,,1731751192181.697cd6a363e514bff0d4ec954fc4034b., hostname=037c58a19ddb,37267,1731751077844, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=697cd6a363e514bff0d4ec954fc4034b, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:00:07,641 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731751192181.697cd6a363e514bff0d4ec954fc4034b., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=697cd6a363e514bff0d4ec954fc4034b, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:00:07,641 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] ipc.CallRunner(138): callId: 114 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36058 deadline: 1731751267633, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=697cd6a363e514bff0d4ec954fc4034b, server=037c58a19ddb,37267,1731751077844 2024-11-16T10:00:07,641 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731751192181.697cd6a363e514bff0d4ec954fc4034b., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 because the exception is null or not the one we care about 2024-11-16T10:00:07,641 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731751192181.697cd6a363e514bff0d4ec954fc4034b., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 , the old value is region=TestAcidGuarantees,,1731751192181.697cd6a363e514bff0d4ec954fc4034b., hostname=037c58a19ddb,37267,1731751077844, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=697cd6a363e514bff0d4ec954fc4034b, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:00:07,641 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731751192181.697cd6a363e514bff0d4ec954fc4034b., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=697cd6a363e514bff0d4ec954fc4034b, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:00:07,641 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731751192181.697cd6a363e514bff0d4ec954fc4034b., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 because the exception is null or not the one we care about 2024-11-16T10:00:07,646 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=273 (bloomFilter=true), to=hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/697cd6a363e514bff0d4ec954fc4034b/.tmp/A/86ab58c9d50f47258676a412736ef737 2024-11-16T10:00:07,654 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/697cd6a363e514bff0d4ec954fc4034b/.tmp/B/a2aa066446a24730ae0703b47b429d8d is 50, key is test_row_0/B:col10/1731751206062/Put/seqid=0 2024-11-16T10:00:07,658 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37591 is added to blk_1073742323_1499 (size=12301) 2024-11-16T10:00:07,659 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=273 (bloomFilter=true), to=hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/697cd6a363e514bff0d4ec954fc4034b/.tmp/B/a2aa066446a24730ae0703b47b429d8d 2024-11-16T10:00:07,670 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/697cd6a363e514bff0d4ec954fc4034b/.tmp/C/22cb10ec5fd249f3ba682ac831965d6d is 50, key is test_row_0/C:col10/1731751206062/Put/seqid=0 2024-11-16T10:00:07,678 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37591 is added to blk_1073742324_1500 (size=12301) 2024-11-16T10:00:07,681 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=273 (bloomFilter=true), to=hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/697cd6a363e514bff0d4ec954fc4034b/.tmp/C/22cb10ec5fd249f3ba682ac831965d6d 2024-11-16T10:00:07,687 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/697cd6a363e514bff0d4ec954fc4034b/.tmp/A/86ab58c9d50f47258676a412736ef737 as hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/697cd6a363e514bff0d4ec954fc4034b/A/86ab58c9d50f47258676a412736ef737 2024-11-16T10:00:07,694 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/697cd6a363e514bff0d4ec954fc4034b/A/86ab58c9d50f47258676a412736ef737, entries=200, sequenceid=273, filesize=14.4 K 2024-11-16T10:00:07,695 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/697cd6a363e514bff0d4ec954fc4034b/.tmp/B/a2aa066446a24730ae0703b47b429d8d as hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/697cd6a363e514bff0d4ec954fc4034b/B/a2aa066446a24730ae0703b47b429d8d 2024-11-16T10:00:07,708 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/697cd6a363e514bff0d4ec954fc4034b/B/a2aa066446a24730ae0703b47b429d8d, entries=150, sequenceid=273, filesize=12.0 K 2024-11-16T10:00:07,709 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/697cd6a363e514bff0d4ec954fc4034b/.tmp/C/22cb10ec5fd249f3ba682ac831965d6d as hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/697cd6a363e514bff0d4ec954fc4034b/C/22cb10ec5fd249f3ba682ac831965d6d 2024-11-16T10:00:07,714 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/697cd6a363e514bff0d4ec954fc4034b/C/22cb10ec5fd249f3ba682ac831965d6d, entries=150, sequenceid=273, filesize=12.0 K 2024-11-16T10:00:07,714 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~147.60 KB/151140, heapSize ~387.42 KB/396720, currentSize=53.67 KB/54960 for 697cd6a363e514bff0d4ec954fc4034b in 484ms, sequenceid=273, compaction requested=true 2024-11-16T10:00:07,715 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 697cd6a363e514bff0d4ec954fc4034b: 2024-11-16T10:00:07,715 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 697cd6a363e514bff0d4ec954fc4034b:A, priority=-2147483648, current under compaction store size is 1 2024-11-16T10:00:07,715 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-16T10:00:07,715 DEBUG [RS:0;037c58a19ddb:37267-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-16T10:00:07,715 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 697cd6a363e514bff0d4ec954fc4034b:B, priority=-2147483648, current under compaction store size is 2 2024-11-16T10:00:07,715 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-16T10:00:07,715 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 697cd6a363e514bff0d4ec954fc4034b:C, priority=-2147483648, current under compaction store size is 3 2024-11-16T10:00:07,715 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-16T10:00:07,715 DEBUG [RS:0;037c58a19ddb:37267-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-16T10:00:07,717 DEBUG [RS:0;037c58a19ddb:37267-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 51502 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-16T10:00:07,717 DEBUG [RS:0;037c58a19ddb:37267-shortCompactions-0 {}] regionserver.HStore(1541): 697cd6a363e514bff0d4ec954fc4034b/A is initiating minor compaction (all files) 2024-11-16T10:00:07,717 INFO [RS:0;037c58a19ddb:37267-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 697cd6a363e514bff0d4ec954fc4034b/A in TestAcidGuarantees,,1731751192181.697cd6a363e514bff0d4ec954fc4034b. 2024-11-16T10:00:07,717 INFO [RS:0;037c58a19ddb:37267-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/697cd6a363e514bff0d4ec954fc4034b/A/aa422c222ac1447d8325f707092eaf7e, hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/697cd6a363e514bff0d4ec954fc4034b/A/3cecc1d82dd343439dbccdd090ece5fd, hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/697cd6a363e514bff0d4ec954fc4034b/A/5b4e2ff342e54bdc84037fe744487a35, hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/697cd6a363e514bff0d4ec954fc4034b/A/86ab58c9d50f47258676a412736ef737] into tmpdir=hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/697cd6a363e514bff0d4ec954fc4034b/.tmp, totalSize=50.3 K 2024-11-16T10:00:07,717 DEBUG [RS:0;037c58a19ddb:37267-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 49062 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-16T10:00:07,717 DEBUG [RS:0;037c58a19ddb:37267-shortCompactions-0 {}] compactions.Compactor(225): Compacting aa422c222ac1447d8325f707092eaf7e, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=208, earliestPutTs=1731751203690 2024-11-16T10:00:07,717 DEBUG [RS:0;037c58a19ddb:37267-longCompactions-0 {}] regionserver.HStore(1541): 697cd6a363e514bff0d4ec954fc4034b/B is initiating minor compaction (all files) 2024-11-16T10:00:07,717 INFO [RS:0;037c58a19ddb:37267-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 697cd6a363e514bff0d4ec954fc4034b/B in TestAcidGuarantees,,1731751192181.697cd6a363e514bff0d4ec954fc4034b. 2024-11-16T10:00:07,717 INFO [RS:0;037c58a19ddb:37267-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/697cd6a363e514bff0d4ec954fc4034b/B/090ecb975fe84165b550dd5f4def4066, hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/697cd6a363e514bff0d4ec954fc4034b/B/df81870aa751418da08296e961c67ac3, hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/697cd6a363e514bff0d4ec954fc4034b/B/8048b8cfb710419bb4932c2b21933a5c, hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/697cd6a363e514bff0d4ec954fc4034b/B/a2aa066446a24730ae0703b47b429d8d] into tmpdir=hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/697cd6a363e514bff0d4ec954fc4034b/.tmp, totalSize=47.9 K 2024-11-16T10:00:07,717 DEBUG [RS:0;037c58a19ddb:37267-shortCompactions-0 {}] compactions.Compactor(225): Compacting 3cecc1d82dd343439dbccdd090ece5fd, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=233, earliestPutTs=1731751204930 2024-11-16T10:00:07,718 DEBUG [RS:0;037c58a19ddb:37267-longCompactions-0 {}] compactions.Compactor(225): Compacting 090ecb975fe84165b550dd5f4def4066, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=208, earliestPutTs=1731751203690 2024-11-16T10:00:07,718 DEBUG [RS:0;037c58a19ddb:37267-shortCompactions-0 {}] compactions.Compactor(225): Compacting 5b4e2ff342e54bdc84037fe744487a35, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=248, earliestPutTs=1731751205361 2024-11-16T10:00:07,718 DEBUG [RS:0;037c58a19ddb:37267-longCompactions-0 {}] compactions.Compactor(225): Compacting df81870aa751418da08296e961c67ac3, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=233, earliestPutTs=1731751204930 2024-11-16T10:00:07,718 DEBUG [RS:0;037c58a19ddb:37267-shortCompactions-0 {}] compactions.Compactor(225): Compacting 86ab58c9d50f47258676a412736ef737, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=273, earliestPutTs=1731751206062 2024-11-16T10:00:07,719 DEBUG [RS:0;037c58a19ddb:37267-longCompactions-0 {}] compactions.Compactor(225): Compacting 8048b8cfb710419bb4932c2b21933a5c, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=248, earliestPutTs=1731751205361 2024-11-16T10:00:07,719 DEBUG [RS:0;037c58a19ddb:37267-longCompactions-0 {}] compactions.Compactor(225): Compacting a2aa066446a24730ae0703b47b429d8d, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=273, earliestPutTs=1731751206062 2024-11-16T10:00:07,731 INFO [RS:0;037c58a19ddb:37267-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 697cd6a363e514bff0d4ec954fc4034b#A#compaction#427 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-16T10:00:07,731 INFO [RS:0;037c58a19ddb:37267-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 697cd6a363e514bff0d4ec954fc4034b#B#compaction#428 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-16T10:00:07,731 DEBUG [RS:0;037c58a19ddb:37267-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/697cd6a363e514bff0d4ec954fc4034b/.tmp/A/7795a15b11b2402dbd7704ab7054d526 is 50, key is test_row_0/A:col10/1731751206062/Put/seqid=0 2024-11-16T10:00:07,731 DEBUG [RS:0;037c58a19ddb:37267-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/697cd6a363e514bff0d4ec954fc4034b/.tmp/B/d6f9eea4334549a18e19ffa0b74c53ac is 50, key is test_row_0/B:col10/1731751206062/Put/seqid=0 2024-11-16T10:00:07,744 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37591 is added to blk_1073742325_1501 (size=12745) 2024-11-16T10:00:07,747 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=37267 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=130 2024-11-16T10:00:07,748 DEBUG [RS_FLUSH_OPERATIONS-regionserver/037c58a19ddb:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731751192181.697cd6a363e514bff0d4ec954fc4034b. 2024-11-16T10:00:07,748 INFO [RS_FLUSH_OPERATIONS-regionserver/037c58a19ddb:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HRegion(2902): Flushing 697cd6a363e514bff0d4ec954fc4034b 3/3 column families, dataSize=53.67 KB heapSize=141.38 KB 2024-11-16T10:00:07,750 DEBUG [RS:0;037c58a19ddb:37267-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/697cd6a363e514bff0d4ec954fc4034b/.tmp/B/d6f9eea4334549a18e19ffa0b74c53ac as hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/697cd6a363e514bff0d4ec954fc4034b/B/d6f9eea4334549a18e19ffa0b74c53ac 2024-11-16T10:00:07,758 INFO [RS:0;037c58a19ddb:37267-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 697cd6a363e514bff0d4ec954fc4034b/B of 697cd6a363e514bff0d4ec954fc4034b into d6f9eea4334549a18e19ffa0b74c53ac(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-16T10:00:07,758 DEBUG [RS:0;037c58a19ddb:37267-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 697cd6a363e514bff0d4ec954fc4034b: 2024-11-16T10:00:07,758 INFO [RS:0;037c58a19ddb:37267-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731751192181.697cd6a363e514bff0d4ec954fc4034b., storeName=697cd6a363e514bff0d4ec954fc4034b/B, priority=12, startTime=1731751207715; duration=0sec 2024-11-16T10:00:07,758 DEBUG [RS:0;037c58a19ddb:37267-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-16T10:00:07,758 DEBUG [RS:0;037c58a19ddb:37267-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 697cd6a363e514bff0d4ec954fc4034b:B 2024-11-16T10:00:07,758 DEBUG [RS:0;037c58a19ddb:37267-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-16T10:00:07,759 DEBUG [RS:0;037c58a19ddb:37267-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 49062 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-16T10:00:07,759 DEBUG [RS:0;037c58a19ddb:37267-longCompactions-0 {}] regionserver.HStore(1541): 697cd6a363e514bff0d4ec954fc4034b/C is initiating minor compaction (all files) 2024-11-16T10:00:07,759 INFO [RS:0;037c58a19ddb:37267-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 697cd6a363e514bff0d4ec954fc4034b/C in TestAcidGuarantees,,1731751192181.697cd6a363e514bff0d4ec954fc4034b. 2024-11-16T10:00:07,759 INFO [RS:0;037c58a19ddb:37267-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/697cd6a363e514bff0d4ec954fc4034b/C/cb29188c3e284c5aafb352c461876405, hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/697cd6a363e514bff0d4ec954fc4034b/C/1a1514cb66b8456bab35a7ce294b1663, hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/697cd6a363e514bff0d4ec954fc4034b/C/699da832fda74291b53ee546204927b7, hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/697cd6a363e514bff0d4ec954fc4034b/C/22cb10ec5fd249f3ba682ac831965d6d] into tmpdir=hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/697cd6a363e514bff0d4ec954fc4034b/.tmp, totalSize=47.9 K 2024-11-16T10:00:07,760 DEBUG [RS:0;037c58a19ddb:37267-longCompactions-0 {}] compactions.Compactor(225): Compacting cb29188c3e284c5aafb352c461876405, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=208, earliestPutTs=1731751203690 2024-11-16T10:00:07,760 DEBUG [RS:0;037c58a19ddb:37267-longCompactions-0 {}] compactions.Compactor(225): Compacting 1a1514cb66b8456bab35a7ce294b1663, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=233, earliestPutTs=1731751204930 2024-11-16T10:00:07,760 DEBUG [RS:0;037c58a19ddb:37267-longCompactions-0 {}] compactions.Compactor(225): Compacting 699da832fda74291b53ee546204927b7, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=248, earliestPutTs=1731751205361 2024-11-16T10:00:07,761 DEBUG [RS:0;037c58a19ddb:37267-longCompactions-0 {}] compactions.Compactor(225): Compacting 22cb10ec5fd249f3ba682ac831965d6d, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=273, earliestPutTs=1731751206062 2024-11-16T10:00:07,763 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37877 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=129 2024-11-16T10:00:07,782 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37591 is added to blk_1073742326_1502 (size=12745) 2024-11-16T10:00:07,789 DEBUG [RS:0;037c58a19ddb:37267-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/697cd6a363e514bff0d4ec954fc4034b/.tmp/A/7795a15b11b2402dbd7704ab7054d526 as hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/697cd6a363e514bff0d4ec954fc4034b/A/7795a15b11b2402dbd7704ab7054d526 2024-11-16T10:00:07,794 INFO [RS:0;037c58a19ddb:37267-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 697cd6a363e514bff0d4ec954fc4034b/A of 697cd6a363e514bff0d4ec954fc4034b into 7795a15b11b2402dbd7704ab7054d526(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-16T10:00:07,794 DEBUG [RS:0;037c58a19ddb:37267-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 697cd6a363e514bff0d4ec954fc4034b: 2024-11-16T10:00:07,794 INFO [RS:0;037c58a19ddb:37267-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731751192181.697cd6a363e514bff0d4ec954fc4034b., storeName=697cd6a363e514bff0d4ec954fc4034b/A, priority=12, startTime=1731751207715; duration=0sec 2024-11-16T10:00:07,794 DEBUG [RS:0;037c58a19ddb:37267-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-16T10:00:07,794 DEBUG [RS:0;037c58a19ddb:37267-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 697cd6a363e514bff0d4ec954fc4034b:A 2024-11-16T10:00:07,798 DEBUG [RS_FLUSH_OPERATIONS-regionserver/037c58a19ddb:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/697cd6a363e514bff0d4ec954fc4034b/.tmp/A/a0ee859be0c44f4b8e62931f139ef719 is 50, key is test_row_0/A:col10/1731751207290/Put/seqid=0 2024-11-16T10:00:07,806 INFO [RS:0;037c58a19ddb:37267-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 697cd6a363e514bff0d4ec954fc4034b#C#compaction#430 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-16T10:00:07,807 DEBUG [RS:0;037c58a19ddb:37267-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/697cd6a363e514bff0d4ec954fc4034b/.tmp/C/8afbac04ec764dc4b9dbddce6d7afb77 is 50, key is test_row_0/C:col10/1731751206062/Put/seqid=0 2024-11-16T10:00:07,819 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37591 is added to blk_1073742328_1504 (size=12745) 2024-11-16T10:00:07,819 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37591 is added to blk_1073742327_1503 (size=12301) 2024-11-16T10:00:07,820 INFO [RS_FLUSH_OPERATIONS-regionserver/037c58a19ddb:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=284 (bloomFilter=true), to=hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/697cd6a363e514bff0d4ec954fc4034b/.tmp/A/a0ee859be0c44f4b8e62931f139ef719 2024-11-16T10:00:07,828 DEBUG [RS:0;037c58a19ddb:37267-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/697cd6a363e514bff0d4ec954fc4034b/.tmp/C/8afbac04ec764dc4b9dbddce6d7afb77 as hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/697cd6a363e514bff0d4ec954fc4034b/C/8afbac04ec764dc4b9dbddce6d7afb77 2024-11-16T10:00:07,835 DEBUG [RS_FLUSH_OPERATIONS-regionserver/037c58a19ddb:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/697cd6a363e514bff0d4ec954fc4034b/.tmp/B/a70bf22e5f60433089494a2d1988999b is 50, key is test_row_0/B:col10/1731751207290/Put/seqid=0 2024-11-16T10:00:07,842 INFO [RS:0;037c58a19ddb:37267-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 697cd6a363e514bff0d4ec954fc4034b/C of 697cd6a363e514bff0d4ec954fc4034b into 8afbac04ec764dc4b9dbddce6d7afb77(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-16T10:00:07,842 DEBUG [RS:0;037c58a19ddb:37267-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 697cd6a363e514bff0d4ec954fc4034b: 2024-11-16T10:00:07,842 INFO [RS:0;037c58a19ddb:37267-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731751192181.697cd6a363e514bff0d4ec954fc4034b., storeName=697cd6a363e514bff0d4ec954fc4034b/C, priority=12, startTime=1731751207715; duration=0sec 2024-11-16T10:00:07,842 DEBUG [RS:0;037c58a19ddb:37267-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-16T10:00:07,842 DEBUG [RS:0;037c58a19ddb:37267-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 697cd6a363e514bff0d4ec954fc4034b:C 2024-11-16T10:00:07,859 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37591 is added to blk_1073742329_1505 (size=12301) 2024-11-16T10:00:07,938 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] regionserver.HRegion(8855): Flush requested on 697cd6a363e514bff0d4ec954fc4034b 2024-11-16T10:00:07,938 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731751192181.697cd6a363e514bff0d4ec954fc4034b. as already flushing 2024-11-16T10:00:08,008 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=697cd6a363e514bff0d4ec954fc4034b, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T10:00:08,008 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36086 deadline: 1731751267993, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=697cd6a363e514bff0d4ec954fc4034b, server=037c58a19ddb,37267,1731751077844 2024-11-16T10:00:08,008 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=697cd6a363e514bff0d4ec954fc4034b, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T10:00:08,008 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] ipc.CallRunner(138): callId: 111 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36012 deadline: 1731751267994, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=697cd6a363e514bff0d4ec954fc4034b, server=037c58a19ddb,37267,1731751077844 2024-11-16T10:00:08,008 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=697cd6a363e514bff0d4ec954fc4034b, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T10:00:08,008 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731751192181.697cd6a363e514bff0d4ec954fc4034b., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 , the old value is region=TestAcidGuarantees,,1731751192181.697cd6a363e514bff0d4ec954fc4034b., hostname=037c58a19ddb,37267,1731751077844, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=697cd6a363e514bff0d4ec954fc4034b, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:00:08,008 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731751192181.697cd6a363e514bff0d4ec954fc4034b., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=697cd6a363e514bff0d4ec954fc4034b, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:00:08,008 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731751192181.697cd6a363e514bff0d4ec954fc4034b., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 because the exception is null or not the one we care about 2024-11-16T10:00:08,008 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] ipc.CallRunner(138): callId: 105 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36022 deadline: 1731751267996, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=697cd6a363e514bff0d4ec954fc4034b, server=037c58a19ddb,37267,1731751077844 2024-11-16T10:00:08,009 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=697cd6a363e514bff0d4ec954fc4034b, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T10:00:08,009 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731751192181.697cd6a363e514bff0d4ec954fc4034b., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 , the old value is region=TestAcidGuarantees,,1731751192181.697cd6a363e514bff0d4ec954fc4034b., hostname=037c58a19ddb,37267,1731751077844, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=697cd6a363e514bff0d4ec954fc4034b, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:00:08,009 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731751192181.697cd6a363e514bff0d4ec954fc4034b., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 , the old value is region=TestAcidGuarantees,,1731751192181.697cd6a363e514bff0d4ec954fc4034b., hostname=037c58a19ddb,37267,1731751077844, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=697cd6a363e514bff0d4ec954fc4034b, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:00:08,009 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731751192181.697cd6a363e514bff0d4ec954fc4034b., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=697cd6a363e514bff0d4ec954fc4034b, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:00:08,009 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] ipc.CallRunner(138): callId: 120 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36058 deadline: 1731751267999, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=697cd6a363e514bff0d4ec954fc4034b, server=037c58a19ddb,37267,1731751077844 2024-11-16T10:00:08,009 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731751192181.697cd6a363e514bff0d4ec954fc4034b., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=697cd6a363e514bff0d4ec954fc4034b, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:00:08,009 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731751192181.697cd6a363e514bff0d4ec954fc4034b., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 because the exception is null or not the one we care about 2024-11-16T10:00:08,009 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731751192181.697cd6a363e514bff0d4ec954fc4034b., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 because the exception is null or not the one we care about 2024-11-16T10:00:08,009 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731751192181.697cd6a363e514bff0d4ec954fc4034b., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 , the old value is region=TestAcidGuarantees,,1731751192181.697cd6a363e514bff0d4ec954fc4034b., hostname=037c58a19ddb,37267,1731751077844, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=697cd6a363e514bff0d4ec954fc4034b, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:00:08,009 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731751192181.697cd6a363e514bff0d4ec954fc4034b., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=697cd6a363e514bff0d4ec954fc4034b, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:00:08,009 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731751192181.697cd6a363e514bff0d4ec954fc4034b., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 because the exception is null or not the one we care about 2024-11-16T10:00:08,120 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=697cd6a363e514bff0d4ec954fc4034b, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T10:00:08,120 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36086 deadline: 1731751268113, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=697cd6a363e514bff0d4ec954fc4034b, server=037c58a19ddb,37267,1731751077844 2024-11-16T10:00:08,120 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=697cd6a363e514bff0d4ec954fc4034b, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T10:00:08,121 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731751192181.697cd6a363e514bff0d4ec954fc4034b., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 , the old value is region=TestAcidGuarantees,,1731751192181.697cd6a363e514bff0d4ec954fc4034b., hostname=037c58a19ddb,37267,1731751077844, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=697cd6a363e514bff0d4ec954fc4034b, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:00:08,121 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731751192181.697cd6a363e514bff0d4ec954fc4034b., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=697cd6a363e514bff0d4ec954fc4034b, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:00:08,121 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] ipc.CallRunner(138): callId: 106 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36022 deadline: 1731751268113, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=697cd6a363e514bff0d4ec954fc4034b, server=037c58a19ddb,37267,1731751077844 2024-11-16T10:00:08,121 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731751192181.697cd6a363e514bff0d4ec954fc4034b., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 because the exception is null or not the one we care about 2024-11-16T10:00:08,121 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=697cd6a363e514bff0d4ec954fc4034b, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T10:00:08,121 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731751192181.697cd6a363e514bff0d4ec954fc4034b., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 , the old value is region=TestAcidGuarantees,,1731751192181.697cd6a363e514bff0d4ec954fc4034b., hostname=037c58a19ddb,37267,1731751077844, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=697cd6a363e514bff0d4ec954fc4034b, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:00:08,121 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] ipc.CallRunner(138): callId: 112 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36012 deadline: 1731751268113, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=697cd6a363e514bff0d4ec954fc4034b, server=037c58a19ddb,37267,1731751077844 2024-11-16T10:00:08,121 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731751192181.697cd6a363e514bff0d4ec954fc4034b., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=697cd6a363e514bff0d4ec954fc4034b, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:00:08,121 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731751192181.697cd6a363e514bff0d4ec954fc4034b., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 because the exception is null or not the one we care about 2024-11-16T10:00:08,121 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=697cd6a363e514bff0d4ec954fc4034b, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T10:00:08,121 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731751192181.697cd6a363e514bff0d4ec954fc4034b., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 , the old value is region=TestAcidGuarantees,,1731751192181.697cd6a363e514bff0d4ec954fc4034b., hostname=037c58a19ddb,37267,1731751077844, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=697cd6a363e514bff0d4ec954fc4034b, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:00:08,121 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] ipc.CallRunner(138): callId: 121 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36058 deadline: 1731751268113, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=697cd6a363e514bff0d4ec954fc4034b, server=037c58a19ddb,37267,1731751077844 2024-11-16T10:00:08,121 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731751192181.697cd6a363e514bff0d4ec954fc4034b., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=697cd6a363e514bff0d4ec954fc4034b, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:00:08,121 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731751192181.697cd6a363e514bff0d4ec954fc4034b., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 because the exception is null or not the one we care about 2024-11-16T10:00:08,121 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731751192181.697cd6a363e514bff0d4ec954fc4034b., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 , the old value is region=TestAcidGuarantees,,1731751192181.697cd6a363e514bff0d4ec954fc4034b., hostname=037c58a19ddb,37267,1731751077844, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=697cd6a363e514bff0d4ec954fc4034b, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:00:08,121 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731751192181.697cd6a363e514bff0d4ec954fc4034b., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=697cd6a363e514bff0d4ec954fc4034b, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:00:08,121 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731751192181.697cd6a363e514bff0d4ec954fc4034b., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 because the exception is null or not the one we care about 2024-11-16T10:00:08,261 INFO [RS_FLUSH_OPERATIONS-regionserver/037c58a19ddb:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=284 (bloomFilter=true), to=hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/697cd6a363e514bff0d4ec954fc4034b/.tmp/B/a70bf22e5f60433089494a2d1988999b 2024-11-16T10:00:08,268 DEBUG [RS_FLUSH_OPERATIONS-regionserver/037c58a19ddb:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/697cd6a363e514bff0d4ec954fc4034b/.tmp/C/475bfdd4b2ed4ab9b2d86b58ceb856f3 is 50, key is test_row_0/C:col10/1731751207290/Put/seqid=0 2024-11-16T10:00:08,272 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37591 is added to blk_1073742330_1506 (size=12301) 2024-11-16T10:00:08,272 INFO [RS_FLUSH_OPERATIONS-regionserver/037c58a19ddb:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=284 (bloomFilter=true), to=hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/697cd6a363e514bff0d4ec954fc4034b/.tmp/C/475bfdd4b2ed4ab9b2d86b58ceb856f3 2024-11-16T10:00:08,273 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37877 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=129 2024-11-16T10:00:08,277 DEBUG [RS_FLUSH_OPERATIONS-regionserver/037c58a19ddb:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/697cd6a363e514bff0d4ec954fc4034b/.tmp/A/a0ee859be0c44f4b8e62931f139ef719 as hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/697cd6a363e514bff0d4ec954fc4034b/A/a0ee859be0c44f4b8e62931f139ef719 2024-11-16T10:00:08,282 INFO [RS_FLUSH_OPERATIONS-regionserver/037c58a19ddb:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/697cd6a363e514bff0d4ec954fc4034b/A/a0ee859be0c44f4b8e62931f139ef719, entries=150, sequenceid=284, filesize=12.0 K 2024-11-16T10:00:08,283 DEBUG [RS_FLUSH_OPERATIONS-regionserver/037c58a19ddb:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/697cd6a363e514bff0d4ec954fc4034b/.tmp/B/a70bf22e5f60433089494a2d1988999b as hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/697cd6a363e514bff0d4ec954fc4034b/B/a70bf22e5f60433089494a2d1988999b 2024-11-16T10:00:08,286 INFO [RS_FLUSH_OPERATIONS-regionserver/037c58a19ddb:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/697cd6a363e514bff0d4ec954fc4034b/B/a70bf22e5f60433089494a2d1988999b, entries=150, sequenceid=284, filesize=12.0 K 2024-11-16T10:00:08,286 DEBUG [RS_FLUSH_OPERATIONS-regionserver/037c58a19ddb:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/697cd6a363e514bff0d4ec954fc4034b/.tmp/C/475bfdd4b2ed4ab9b2d86b58ceb856f3 as hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/697cd6a363e514bff0d4ec954fc4034b/C/475bfdd4b2ed4ab9b2d86b58ceb856f3 2024-11-16T10:00:08,289 INFO [RS_FLUSH_OPERATIONS-regionserver/037c58a19ddb:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/697cd6a363e514bff0d4ec954fc4034b/C/475bfdd4b2ed4ab9b2d86b58ceb856f3, entries=150, sequenceid=284, filesize=12.0 K 2024-11-16T10:00:08,290 INFO [RS_FLUSH_OPERATIONS-regionserver/037c58a19ddb:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~141.33 KB/144720, currentSize=147.60 KB/151140 for 697cd6a363e514bff0d4ec954fc4034b in 542ms, sequenceid=284, compaction requested=false 2024-11-16T10:00:08,290 DEBUG [RS_FLUSH_OPERATIONS-regionserver/037c58a19ddb:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HRegion(2603): Flush status journal for 697cd6a363e514bff0d4ec954fc4034b: 2024-11-16T10:00:08,290 DEBUG [RS_FLUSH_OPERATIONS-regionserver/037c58a19ddb:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731751192181.697cd6a363e514bff0d4ec954fc4034b. 2024-11-16T10:00:08,290 DEBUG [RS_FLUSH_OPERATIONS-regionserver/037c58a19ddb:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=130 2024-11-16T10:00:08,291 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37877 {}] master.HMaster(4169): Remote procedure done, pid=130 2024-11-16T10:00:08,296 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=130, resume processing ppid=129 2024-11-16T10:00:08,296 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=130, ppid=129, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.1570 sec 2024-11-16T10:00:08,298 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=129, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=129, table=TestAcidGuarantees in 1.1630 sec 2024-11-16T10:00:08,339 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] regionserver.HRegion(8855): Flush requested on 697cd6a363e514bff0d4ec954fc4034b 2024-11-16T10:00:08,340 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-16T10:00:08,340 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 697cd6a363e514bff0d4ec954fc4034b 3/3 column families, dataSize=154.31 KB heapSize=405.05 KB 2024-11-16T10:00:08,345 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/697cd6a363e514bff0d4ec954fc4034b/.tmp/A/844f11e79e4445f49c53f85d0e276e2d is 50, key is test_row_0/A:col10/1731751207992/Put/seqid=0 2024-11-16T10:00:08,349 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37591 is added to blk_1073742331_1507 (size=14741) 2024-11-16T10:00:08,355 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=697cd6a363e514bff0d4ec954fc4034b, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T10:00:08,355 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] ipc.CallRunner(138): callId: 109 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36022 deadline: 1731751268354, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=697cd6a363e514bff0d4ec954fc4034b, server=037c58a19ddb,37267,1731751077844 2024-11-16T10:00:08,355 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=697cd6a363e514bff0d4ec954fc4034b, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T10:00:08,355 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] ipc.CallRunner(138): callId: 124 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36058 deadline: 1731751268354, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=697cd6a363e514bff0d4ec954fc4034b, server=037c58a19ddb,37267,1731751077844 2024-11-16T10:00:08,355 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731751192181.697cd6a363e514bff0d4ec954fc4034b., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 , the old value is region=TestAcidGuarantees,,1731751192181.697cd6a363e514bff0d4ec954fc4034b., hostname=037c58a19ddb,37267,1731751077844, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=697cd6a363e514bff0d4ec954fc4034b, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:00:08,356 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731751192181.697cd6a363e514bff0d4ec954fc4034b., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=697cd6a363e514bff0d4ec954fc4034b, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:00:08,356 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731751192181.697cd6a363e514bff0d4ec954fc4034b., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 because the exception is null or not the one we care about 2024-11-16T10:00:08,356 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731751192181.697cd6a363e514bff0d4ec954fc4034b., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 , the old value is region=TestAcidGuarantees,,1731751192181.697cd6a363e514bff0d4ec954fc4034b., hostname=037c58a19ddb,37267,1731751077844, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=697cd6a363e514bff0d4ec954fc4034b, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:00:08,356 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731751192181.697cd6a363e514bff0d4ec954fc4034b., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=697cd6a363e514bff0d4ec954fc4034b, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:00:08,356 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731751192181.697cd6a363e514bff0d4ec954fc4034b., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 because the exception is null or not the one we care about 2024-11-16T10:00:08,364 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=697cd6a363e514bff0d4ec954fc4034b, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T10:00:08,364 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] ipc.CallRunner(138): callId: 115 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36012 deadline: 1731751268355, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=697cd6a363e514bff0d4ec954fc4034b, server=037c58a19ddb,37267,1731751077844 2024-11-16T10:00:08,364 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=697cd6a363e514bff0d4ec954fc4034b, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T10:00:08,364 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36086 deadline: 1731751268355, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=697cd6a363e514bff0d4ec954fc4034b, server=037c58a19ddb,37267,1731751077844 2024-11-16T10:00:08,364 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731751192181.697cd6a363e514bff0d4ec954fc4034b., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 , the old value is region=TestAcidGuarantees,,1731751192181.697cd6a363e514bff0d4ec954fc4034b., hostname=037c58a19ddb,37267,1731751077844, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=697cd6a363e514bff0d4ec954fc4034b, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:00:08,364 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731751192181.697cd6a363e514bff0d4ec954fc4034b., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=697cd6a363e514bff0d4ec954fc4034b, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:00:08,364 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731751192181.697cd6a363e514bff0d4ec954fc4034b., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 because the exception is null or not the one we care about 2024-11-16T10:00:08,365 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731751192181.697cd6a363e514bff0d4ec954fc4034b., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 , the old value is region=TestAcidGuarantees,,1731751192181.697cd6a363e514bff0d4ec954fc4034b., hostname=037c58a19ddb,37267,1731751077844, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=697cd6a363e514bff0d4ec954fc4034b, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:00:08,365 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731751192181.697cd6a363e514bff0d4ec954fc4034b., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=697cd6a363e514bff0d4ec954fc4034b, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:00:08,365 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731751192181.697cd6a363e514bff0d4ec954fc4034b., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 because the exception is null or not the one we care about 2024-11-16T10:00:08,466 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=697cd6a363e514bff0d4ec954fc4034b, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T10:00:08,466 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] ipc.CallRunner(138): callId: 110 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36022 deadline: 1731751268463, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=697cd6a363e514bff0d4ec954fc4034b, server=037c58a19ddb,37267,1731751077844 2024-11-16T10:00:08,466 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=697cd6a363e514bff0d4ec954fc4034b, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T10:00:08,466 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731751192181.697cd6a363e514bff0d4ec954fc4034b., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 , the old value is region=TestAcidGuarantees,,1731751192181.697cd6a363e514bff0d4ec954fc4034b., hostname=037c58a19ddb,37267,1731751077844, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=697cd6a363e514bff0d4ec954fc4034b, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:00:08,466 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] ipc.CallRunner(138): callId: 125 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36058 deadline: 1731751268463, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=697cd6a363e514bff0d4ec954fc4034b, server=037c58a19ddb,37267,1731751077844 2024-11-16T10:00:08,466 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731751192181.697cd6a363e514bff0d4ec954fc4034b., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=697cd6a363e514bff0d4ec954fc4034b, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:00:08,466 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731751192181.697cd6a363e514bff0d4ec954fc4034b., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 because the exception is null or not the one we care about 2024-11-16T10:00:08,466 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731751192181.697cd6a363e514bff0d4ec954fc4034b., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 , the old value is region=TestAcidGuarantees,,1731751192181.697cd6a363e514bff0d4ec954fc4034b., hostname=037c58a19ddb,37267,1731751077844, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=697cd6a363e514bff0d4ec954fc4034b, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:00:08,466 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731751192181.697cd6a363e514bff0d4ec954fc4034b., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=697cd6a363e514bff0d4ec954fc4034b, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:00:08,466 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731751192181.697cd6a363e514bff0d4ec954fc4034b., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 because the exception is null or not the one we care about 2024-11-16T10:00:08,480 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=697cd6a363e514bff0d4ec954fc4034b, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T10:00:08,480 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] ipc.CallRunner(138): callId: 116 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36012 deadline: 1731751268473, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=697cd6a363e514bff0d4ec954fc4034b, server=037c58a19ddb,37267,1731751077844 2024-11-16T10:00:08,481 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=697cd6a363e514bff0d4ec954fc4034b, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T10:00:08,481 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731751192181.697cd6a363e514bff0d4ec954fc4034b., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 , the old value is region=TestAcidGuarantees,,1731751192181.697cd6a363e514bff0d4ec954fc4034b., hostname=037c58a19ddb,37267,1731751077844, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=697cd6a363e514bff0d4ec954fc4034b, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:00:08,481 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731751192181.697cd6a363e514bff0d4ec954fc4034b., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=697cd6a363e514bff0d4ec954fc4034b, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:00:08,481 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36086 deadline: 1731751268473, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=697cd6a363e514bff0d4ec954fc4034b, server=037c58a19ddb,37267,1731751077844 2024-11-16T10:00:08,481 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731751192181.697cd6a363e514bff0d4ec954fc4034b., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 because the exception is null or not the one we care about 2024-11-16T10:00:08,481 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731751192181.697cd6a363e514bff0d4ec954fc4034b., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 , the old value is region=TestAcidGuarantees,,1731751192181.697cd6a363e514bff0d4ec954fc4034b., hostname=037c58a19ddb,37267,1731751077844, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=697cd6a363e514bff0d4ec954fc4034b, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:00:08,481 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731751192181.697cd6a363e514bff0d4ec954fc4034b., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=697cd6a363e514bff0d4ec954fc4034b, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:00:08,481 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731751192181.697cd6a363e514bff0d4ec954fc4034b., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 because the exception is null or not the one we care about 2024-11-16T10:00:08,679 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=697cd6a363e514bff0d4ec954fc4034b, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T10:00:08,679 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] ipc.CallRunner(138): callId: 111 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36022 deadline: 1731751268673, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=697cd6a363e514bff0d4ec954fc4034b, server=037c58a19ddb,37267,1731751077844 2024-11-16T10:00:08,680 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731751192181.697cd6a363e514bff0d4ec954fc4034b., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 , the old value is region=TestAcidGuarantees,,1731751192181.697cd6a363e514bff0d4ec954fc4034b., hostname=037c58a19ddb,37267,1731751077844, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=697cd6a363e514bff0d4ec954fc4034b, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:00:08,680 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=697cd6a363e514bff0d4ec954fc4034b, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T10:00:08,680 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731751192181.697cd6a363e514bff0d4ec954fc4034b., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=697cd6a363e514bff0d4ec954fc4034b, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:00:08,680 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731751192181.697cd6a363e514bff0d4ec954fc4034b., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 because the exception is null or not the one we care about 2024-11-16T10:00:08,680 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] ipc.CallRunner(138): callId: 126 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36058 deadline: 1731751268673, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=697cd6a363e514bff0d4ec954fc4034b, server=037c58a19ddb,37267,1731751077844 2024-11-16T10:00:08,680 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731751192181.697cd6a363e514bff0d4ec954fc4034b., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 , the old value is region=TestAcidGuarantees,,1731751192181.697cd6a363e514bff0d4ec954fc4034b., hostname=037c58a19ddb,37267,1731751077844, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=697cd6a363e514bff0d4ec954fc4034b, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:00:08,680 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731751192181.697cd6a363e514bff0d4ec954fc4034b., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=697cd6a363e514bff0d4ec954fc4034b, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:00:08,680 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731751192181.697cd6a363e514bff0d4ec954fc4034b., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 because the exception is null or not the one we care about 2024-11-16T10:00:08,690 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=697cd6a363e514bff0d4ec954fc4034b, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T10:00:08,690 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] ipc.CallRunner(138): callId: 117 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36012 deadline: 1731751268683, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=697cd6a363e514bff0d4ec954fc4034b, server=037c58a19ddb,37267,1731751077844 2024-11-16T10:00:08,690 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731751192181.697cd6a363e514bff0d4ec954fc4034b., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 , the old value is region=TestAcidGuarantees,,1731751192181.697cd6a363e514bff0d4ec954fc4034b., hostname=037c58a19ddb,37267,1731751077844, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=697cd6a363e514bff0d4ec954fc4034b, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:00:08,690 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731751192181.697cd6a363e514bff0d4ec954fc4034b., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=697cd6a363e514bff0d4ec954fc4034b, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:00:08,690 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731751192181.697cd6a363e514bff0d4ec954fc4034b., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 because the exception is null or not the one we care about 2024-11-16T10:00:08,699 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=697cd6a363e514bff0d4ec954fc4034b, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T10:00:08,699 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36086 deadline: 1731751268693, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=697cd6a363e514bff0d4ec954fc4034b, server=037c58a19ddb,37267,1731751077844 2024-11-16T10:00:08,699 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731751192181.697cd6a363e514bff0d4ec954fc4034b., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 , the old value is region=TestAcidGuarantees,,1731751192181.697cd6a363e514bff0d4ec954fc4034b., hostname=037c58a19ddb,37267,1731751077844, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=697cd6a363e514bff0d4ec954fc4034b, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:00:08,699 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731751192181.697cd6a363e514bff0d4ec954fc4034b., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=697cd6a363e514bff0d4ec954fc4034b, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:00:08,700 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731751192181.697cd6a363e514bff0d4ec954fc4034b., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 because the exception is null or not the one we care about 2024-11-16T10:00:08,750 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=51.44 KB at sequenceid=313 (bloomFilter=true), to=hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/697cd6a363e514bff0d4ec954fc4034b/.tmp/A/844f11e79e4445f49c53f85d0e276e2d 2024-11-16T10:00:08,757 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/697cd6a363e514bff0d4ec954fc4034b/.tmp/B/3e7b206106e64728ae5589c7ff779a38 is 50, key is test_row_0/B:col10/1731751207992/Put/seqid=0 2024-11-16T10:00:08,761 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37591 is added to blk_1073742332_1508 (size=12301) 2024-11-16T10:00:08,989 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=697cd6a363e514bff0d4ec954fc4034b, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T10:00:08,989 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] ipc.CallRunner(138): callId: 112 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36022 deadline: 1731751268983, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=697cd6a363e514bff0d4ec954fc4034b, server=037c58a19ddb,37267,1731751077844 2024-11-16T10:00:08,990 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=697cd6a363e514bff0d4ec954fc4034b, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T10:00:08,990 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] ipc.CallRunner(138): callId: 127 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36058 deadline: 1731751268983, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=697cd6a363e514bff0d4ec954fc4034b, server=037c58a19ddb,37267,1731751077844 2024-11-16T10:00:08,990 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731751192181.697cd6a363e514bff0d4ec954fc4034b., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 , the old value is region=TestAcidGuarantees,,1731751192181.697cd6a363e514bff0d4ec954fc4034b., hostname=037c58a19ddb,37267,1731751077844, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=697cd6a363e514bff0d4ec954fc4034b, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:00:08,990 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731751192181.697cd6a363e514bff0d4ec954fc4034b., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=697cd6a363e514bff0d4ec954fc4034b, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:00:08,990 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731751192181.697cd6a363e514bff0d4ec954fc4034b., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 because the exception is null or not the one we care about 2024-11-16T10:00:08,990 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731751192181.697cd6a363e514bff0d4ec954fc4034b., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 , the old value is region=TestAcidGuarantees,,1731751192181.697cd6a363e514bff0d4ec954fc4034b., hostname=037c58a19ddb,37267,1731751077844, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=697cd6a363e514bff0d4ec954fc4034b, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:00:08,990 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731751192181.697cd6a363e514bff0d4ec954fc4034b., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=697cd6a363e514bff0d4ec954fc4034b, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:00:08,990 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731751192181.697cd6a363e514bff0d4ec954fc4034b., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 because the exception is null or not the one we care about 2024-11-16T10:00:09,000 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=697cd6a363e514bff0d4ec954fc4034b, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T10:00:09,000 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] ipc.CallRunner(138): callId: 118 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36012 deadline: 1731751268992, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=697cd6a363e514bff0d4ec954fc4034b, server=037c58a19ddb,37267,1731751077844 2024-11-16T10:00:09,000 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731751192181.697cd6a363e514bff0d4ec954fc4034b., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 , the old value is region=TestAcidGuarantees,,1731751192181.697cd6a363e514bff0d4ec954fc4034b., hostname=037c58a19ddb,37267,1731751077844, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=697cd6a363e514bff0d4ec954fc4034b, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:00:09,000 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731751192181.697cd6a363e514bff0d4ec954fc4034b., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=697cd6a363e514bff0d4ec954fc4034b, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:00:09,000 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731751192181.697cd6a363e514bff0d4ec954fc4034b., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 because the exception is null or not the one we care about 2024-11-16T10:00:09,010 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=697cd6a363e514bff0d4ec954fc4034b, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T10:00:09,010 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36086 deadline: 1731751269003, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=697cd6a363e514bff0d4ec954fc4034b, server=037c58a19ddb,37267,1731751077844 2024-11-16T10:00:09,010 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731751192181.697cd6a363e514bff0d4ec954fc4034b., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 , the old value is region=TestAcidGuarantees,,1731751192181.697cd6a363e514bff0d4ec954fc4034b., hostname=037c58a19ddb,37267,1731751077844, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=697cd6a363e514bff0d4ec954fc4034b, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:00:09,010 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731751192181.697cd6a363e514bff0d4ec954fc4034b., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=697cd6a363e514bff0d4ec954fc4034b, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:00:09,010 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731751192181.697cd6a363e514bff0d4ec954fc4034b., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 because the exception is null or not the one we care about 2024-11-16T10:00:09,162 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=51.44 KB at sequenceid=313 (bloomFilter=true), to=hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/697cd6a363e514bff0d4ec954fc4034b/.tmp/B/3e7b206106e64728ae5589c7ff779a38 2024-11-16T10:00:09,170 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/697cd6a363e514bff0d4ec954fc4034b/.tmp/C/1d64ef6ef6ef4156812f479b5da065f6 is 50, key is test_row_0/C:col10/1731751207992/Put/seqid=0 2024-11-16T10:00:09,176 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37591 is added to blk_1073742333_1509 (size=12301) 2024-11-16T10:00:09,180 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=51.44 KB at sequenceid=313 (bloomFilter=true), to=hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/697cd6a363e514bff0d4ec954fc4034b/.tmp/C/1d64ef6ef6ef4156812f479b5da065f6 2024-11-16T10:00:09,185 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/697cd6a363e514bff0d4ec954fc4034b/.tmp/A/844f11e79e4445f49c53f85d0e276e2d as hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/697cd6a363e514bff0d4ec954fc4034b/A/844f11e79e4445f49c53f85d0e276e2d 2024-11-16T10:00:09,189 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/697cd6a363e514bff0d4ec954fc4034b/A/844f11e79e4445f49c53f85d0e276e2d, entries=200, sequenceid=313, filesize=14.4 K 2024-11-16T10:00:09,189 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/697cd6a363e514bff0d4ec954fc4034b/.tmp/B/3e7b206106e64728ae5589c7ff779a38 as hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/697cd6a363e514bff0d4ec954fc4034b/B/3e7b206106e64728ae5589c7ff779a38 2024-11-16T10:00:09,192 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/697cd6a363e514bff0d4ec954fc4034b/B/3e7b206106e64728ae5589c7ff779a38, entries=150, sequenceid=313, filesize=12.0 K 2024-11-16T10:00:09,193 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/697cd6a363e514bff0d4ec954fc4034b/.tmp/C/1d64ef6ef6ef4156812f479b5da065f6 as hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/697cd6a363e514bff0d4ec954fc4034b/C/1d64ef6ef6ef4156812f479b5da065f6 2024-11-16T10:00:09,197 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/697cd6a363e514bff0d4ec954fc4034b/C/1d64ef6ef6ef4156812f479b5da065f6, entries=150, sequenceid=313, filesize=12.0 K 2024-11-16T10:00:09,198 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~154.31 KB/158010, heapSize ~405 KB/414720, currentSize=53.67 KB/54960 for 697cd6a363e514bff0d4ec954fc4034b in 858ms, sequenceid=313, compaction requested=true 2024-11-16T10:00:09,198 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 697cd6a363e514bff0d4ec954fc4034b: 2024-11-16T10:00:09,198 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 697cd6a363e514bff0d4ec954fc4034b:A, priority=-2147483648, current under compaction store size is 1 2024-11-16T10:00:09,198 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-16T10:00:09,198 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 697cd6a363e514bff0d4ec954fc4034b:B, priority=-2147483648, current under compaction store size is 2 2024-11-16T10:00:09,198 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-16T10:00:09,198 DEBUG [RS:0;037c58a19ddb:37267-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-16T10:00:09,198 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 697cd6a363e514bff0d4ec954fc4034b:C, priority=-2147483648, current under compaction store size is 3 2024-11-16T10:00:09,198 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-16T10:00:09,198 DEBUG [RS:0;037c58a19ddb:37267-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-16T10:00:09,199 DEBUG [RS:0;037c58a19ddb:37267-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37347 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-16T10:00:09,199 DEBUG [RS:0;037c58a19ddb:37267-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 39787 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-16T10:00:09,199 DEBUG [RS:0;037c58a19ddb:37267-longCompactions-0 {}] regionserver.HStore(1541): 697cd6a363e514bff0d4ec954fc4034b/B is initiating minor compaction (all files) 2024-11-16T10:00:09,199 DEBUG [RS:0;037c58a19ddb:37267-shortCompactions-0 {}] regionserver.HStore(1541): 697cd6a363e514bff0d4ec954fc4034b/A is initiating minor compaction (all files) 2024-11-16T10:00:09,199 INFO [RS:0;037c58a19ddb:37267-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 697cd6a363e514bff0d4ec954fc4034b/B in TestAcidGuarantees,,1731751192181.697cd6a363e514bff0d4ec954fc4034b. 2024-11-16T10:00:09,199 INFO [RS:0;037c58a19ddb:37267-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 697cd6a363e514bff0d4ec954fc4034b/A in TestAcidGuarantees,,1731751192181.697cd6a363e514bff0d4ec954fc4034b. 2024-11-16T10:00:09,199 INFO [RS:0;037c58a19ddb:37267-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/697cd6a363e514bff0d4ec954fc4034b/B/d6f9eea4334549a18e19ffa0b74c53ac, hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/697cd6a363e514bff0d4ec954fc4034b/B/a70bf22e5f60433089494a2d1988999b, hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/697cd6a363e514bff0d4ec954fc4034b/B/3e7b206106e64728ae5589c7ff779a38] into tmpdir=hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/697cd6a363e514bff0d4ec954fc4034b/.tmp, totalSize=36.5 K 2024-11-16T10:00:09,199 INFO [RS:0;037c58a19ddb:37267-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/697cd6a363e514bff0d4ec954fc4034b/A/7795a15b11b2402dbd7704ab7054d526, hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/697cd6a363e514bff0d4ec954fc4034b/A/a0ee859be0c44f4b8e62931f139ef719, hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/697cd6a363e514bff0d4ec954fc4034b/A/844f11e79e4445f49c53f85d0e276e2d] into tmpdir=hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/697cd6a363e514bff0d4ec954fc4034b/.tmp, totalSize=38.9 K 2024-11-16T10:00:09,199 DEBUG [RS:0;037c58a19ddb:37267-longCompactions-0 {}] compactions.Compactor(225): Compacting d6f9eea4334549a18e19ffa0b74c53ac, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=273, earliestPutTs=1731751206062 2024-11-16T10:00:09,199 DEBUG [RS:0;037c58a19ddb:37267-shortCompactions-0 {}] compactions.Compactor(225): Compacting 7795a15b11b2402dbd7704ab7054d526, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=273, earliestPutTs=1731751206062 2024-11-16T10:00:09,199 DEBUG [RS:0;037c58a19ddb:37267-longCompactions-0 {}] compactions.Compactor(225): Compacting a70bf22e5f60433089494a2d1988999b, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=284, earliestPutTs=1731751207241 2024-11-16T10:00:09,199 DEBUG [RS:0;037c58a19ddb:37267-shortCompactions-0 {}] compactions.Compactor(225): Compacting a0ee859be0c44f4b8e62931f139ef719, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=284, earliestPutTs=1731751207241 2024-11-16T10:00:09,200 DEBUG [RS:0;037c58a19ddb:37267-longCompactions-0 {}] compactions.Compactor(225): Compacting 3e7b206106e64728ae5589c7ff779a38, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=313, earliestPutTs=1731751207992 2024-11-16T10:00:09,200 DEBUG [RS:0;037c58a19ddb:37267-shortCompactions-0 {}] compactions.Compactor(225): Compacting 844f11e79e4445f49c53f85d0e276e2d, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=313, earliestPutTs=1731751207992 2024-11-16T10:00:09,206 INFO [RS:0;037c58a19ddb:37267-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 697cd6a363e514bff0d4ec954fc4034b#A#compaction#436 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-16T10:00:09,206 DEBUG [RS:0;037c58a19ddb:37267-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/697cd6a363e514bff0d4ec954fc4034b/.tmp/A/c3c82a2503ed413f9db2739f818730bf is 50, key is test_row_0/A:col10/1731751207992/Put/seqid=0 2024-11-16T10:00:09,207 INFO [RS:0;037c58a19ddb:37267-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 697cd6a363e514bff0d4ec954fc4034b#B#compaction#437 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-16T10:00:09,208 DEBUG [RS:0;037c58a19ddb:37267-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/697cd6a363e514bff0d4ec954fc4034b/.tmp/B/6cdb974236134267a7e4d6f4f42abaa2 is 50, key is test_row_0/B:col10/1731751207992/Put/seqid=0 2024-11-16T10:00:09,212 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37591 is added to blk_1073742334_1510 (size=12847) 2024-11-16T10:00:09,219 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37591 is added to blk_1073742335_1511 (size=12847) 2024-11-16T10:00:09,283 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37877 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=129 2024-11-16T10:00:09,283 INFO [RPCClient-NioEventLoopGroup-4-10 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-16T10:00:09,284 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37877 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-16T10:00:09,285 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37877 {}] procedure2.ProcedureExecutor(1139): Stored pid=131, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=131, table=TestAcidGuarantees 2024-11-16T10:00:09,285 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37877 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=131 2024-11-16T10:00:09,286 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=131, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=131, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-16T10:00:09,286 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=131, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=131, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-16T10:00:09,286 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=132, ppid=131, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-16T10:00:09,393 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37877 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=131 2024-11-16T10:00:09,439 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=37267 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=132 2024-11-16T10:00:09,439 DEBUG [RS_FLUSH_OPERATIONS-regionserver/037c58a19ddb:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731751192181.697cd6a363e514bff0d4ec954fc4034b. 2024-11-16T10:00:09,439 INFO [RS_FLUSH_OPERATIONS-regionserver/037c58a19ddb:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.HRegion(2902): Flushing 697cd6a363e514bff0d4ec954fc4034b 3/3 column families, dataSize=53.67 KB heapSize=141.38 KB 2024-11-16T10:00:09,444 DEBUG [RS_FLUSH_OPERATIONS-regionserver/037c58a19ddb:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/697cd6a363e514bff0d4ec954fc4034b/.tmp/A/56afeb41771c4583b16aa0eb393faa0a is 50, key is test_row_0/A:col10/1731751208352/Put/seqid=0 2024-11-16T10:00:09,451 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37591 is added to blk_1073742336_1512 (size=12301) 2024-11-16T10:00:09,452 INFO [RS_FLUSH_OPERATIONS-regionserver/037c58a19ddb:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=324 (bloomFilter=true), to=hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/697cd6a363e514bff0d4ec954fc4034b/.tmp/A/56afeb41771c4583b16aa0eb393faa0a 2024-11-16T10:00:09,461 DEBUG [RS_FLUSH_OPERATIONS-regionserver/037c58a19ddb:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/697cd6a363e514bff0d4ec954fc4034b/.tmp/B/c5f885ecf9af4420be16cd8dd42d29cf is 50, key is test_row_0/B:col10/1731751208352/Put/seqid=0 2024-11-16T10:00:09,474 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37591 is added to blk_1073742337_1513 (size=12301) 2024-11-16T10:00:09,494 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] regionserver.HRegion(8855): Flush requested on 697cd6a363e514bff0d4ec954fc4034b 2024-11-16T10:00:09,494 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731751192181.697cd6a363e514bff0d4ec954fc4034b. as already flushing 2024-11-16T10:00:09,561 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=697cd6a363e514bff0d4ec954fc4034b, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T10:00:09,562 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36086 deadline: 1731751269558, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=697cd6a363e514bff0d4ec954fc4034b, server=037c58a19ddb,37267,1731751077844 2024-11-16T10:00:09,562 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731751192181.697cd6a363e514bff0d4ec954fc4034b., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 , the old value is region=TestAcidGuarantees,,1731751192181.697cd6a363e514bff0d4ec954fc4034b., hostname=037c58a19ddb,37267,1731751077844, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=697cd6a363e514bff0d4ec954fc4034b, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:00:09,562 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731751192181.697cd6a363e514bff0d4ec954fc4034b., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=697cd6a363e514bff0d4ec954fc4034b, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:00:09,562 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731751192181.697cd6a363e514bff0d4ec954fc4034b., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 because the exception is null or not the one we care about 2024-11-16T10:00:09,569 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=697cd6a363e514bff0d4ec954fc4034b, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T10:00:09,570 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] ipc.CallRunner(138): callId: 124 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36012 deadline: 1731751269559, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=697cd6a363e514bff0d4ec954fc4034b, server=037c58a19ddb,37267,1731751077844 2024-11-16T10:00:09,570 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731751192181.697cd6a363e514bff0d4ec954fc4034b., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 , the old value is region=TestAcidGuarantees,,1731751192181.697cd6a363e514bff0d4ec954fc4034b., hostname=037c58a19ddb,37267,1731751077844, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=697cd6a363e514bff0d4ec954fc4034b, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:00:09,570 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731751192181.697cd6a363e514bff0d4ec954fc4034b., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=697cd6a363e514bff0d4ec954fc4034b, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:00:09,570 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731751192181.697cd6a363e514bff0d4ec954fc4034b., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 because the exception is null or not the one we care about 2024-11-16T10:00:09,572 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=697cd6a363e514bff0d4ec954fc4034b, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T10:00:09,573 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] ipc.CallRunner(138): callId: 133 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36058 deadline: 1731751269561, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=697cd6a363e514bff0d4ec954fc4034b, server=037c58a19ddb,37267,1731751077844 2024-11-16T10:00:09,573 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=697cd6a363e514bff0d4ec954fc4034b, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T10:00:09,573 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731751192181.697cd6a363e514bff0d4ec954fc4034b., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 , the old value is region=TestAcidGuarantees,,1731751192181.697cd6a363e514bff0d4ec954fc4034b., hostname=037c58a19ddb,37267,1731751077844, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=697cd6a363e514bff0d4ec954fc4034b, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:00:09,573 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731751192181.697cd6a363e514bff0d4ec954fc4034b., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=697cd6a363e514bff0d4ec954fc4034b, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:00:09,573 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] ipc.CallRunner(138): callId: 122 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36022 deadline: 1731751269562, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=697cd6a363e514bff0d4ec954fc4034b, server=037c58a19ddb,37267,1731751077844 2024-11-16T10:00:09,573 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731751192181.697cd6a363e514bff0d4ec954fc4034b., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 because the exception is null or not the one we care about 2024-11-16T10:00:09,573 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731751192181.697cd6a363e514bff0d4ec954fc4034b., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 , the old value is region=TestAcidGuarantees,,1731751192181.697cd6a363e514bff0d4ec954fc4034b., hostname=037c58a19ddb,37267,1731751077844, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=697cd6a363e514bff0d4ec954fc4034b, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:00:09,573 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731751192181.697cd6a363e514bff0d4ec954fc4034b., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=697cd6a363e514bff0d4ec954fc4034b, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:00:09,573 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731751192181.697cd6a363e514bff0d4ec954fc4034b., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 because the exception is null or not the one we care about 2024-11-16T10:00:09,603 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37877 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=131 2024-11-16T10:00:09,617 DEBUG [RS:0;037c58a19ddb:37267-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/697cd6a363e514bff0d4ec954fc4034b/.tmp/A/c3c82a2503ed413f9db2739f818730bf as hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/697cd6a363e514bff0d4ec954fc4034b/A/c3c82a2503ed413f9db2739f818730bf 2024-11-16T10:00:09,622 INFO [RS:0;037c58a19ddb:37267-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 697cd6a363e514bff0d4ec954fc4034b/A of 697cd6a363e514bff0d4ec954fc4034b into c3c82a2503ed413f9db2739f818730bf(size=12.5 K), total size for store is 12.5 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-16T10:00:09,622 DEBUG [RS:0;037c58a19ddb:37267-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 697cd6a363e514bff0d4ec954fc4034b: 2024-11-16T10:00:09,622 INFO [RS:0;037c58a19ddb:37267-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731751192181.697cd6a363e514bff0d4ec954fc4034b., storeName=697cd6a363e514bff0d4ec954fc4034b/A, priority=13, startTime=1731751209198; duration=0sec 2024-11-16T10:00:09,622 DEBUG [RS:0;037c58a19ddb:37267-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-16T10:00:09,622 DEBUG [RS:0;037c58a19ddb:37267-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 697cd6a363e514bff0d4ec954fc4034b:A 2024-11-16T10:00:09,622 DEBUG [RS:0;037c58a19ddb:37267-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-16T10:00:09,623 DEBUG [RS:0;037c58a19ddb:37267-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37347 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-16T10:00:09,623 DEBUG [RS:0;037c58a19ddb:37267-shortCompactions-0 {}] regionserver.HStore(1541): 697cd6a363e514bff0d4ec954fc4034b/C is initiating minor compaction (all files) 2024-11-16T10:00:09,623 INFO [RS:0;037c58a19ddb:37267-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 697cd6a363e514bff0d4ec954fc4034b/C in TestAcidGuarantees,,1731751192181.697cd6a363e514bff0d4ec954fc4034b. 2024-11-16T10:00:09,623 INFO [RS:0;037c58a19ddb:37267-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/697cd6a363e514bff0d4ec954fc4034b/C/8afbac04ec764dc4b9dbddce6d7afb77, hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/697cd6a363e514bff0d4ec954fc4034b/C/475bfdd4b2ed4ab9b2d86b58ceb856f3, hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/697cd6a363e514bff0d4ec954fc4034b/C/1d64ef6ef6ef4156812f479b5da065f6] into tmpdir=hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/697cd6a363e514bff0d4ec954fc4034b/.tmp, totalSize=36.5 K 2024-11-16T10:00:09,624 DEBUG [RS:0;037c58a19ddb:37267-shortCompactions-0 {}] compactions.Compactor(225): Compacting 8afbac04ec764dc4b9dbddce6d7afb77, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=273, earliestPutTs=1731751206062 2024-11-16T10:00:09,624 DEBUG [RS:0;037c58a19ddb:37267-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/697cd6a363e514bff0d4ec954fc4034b/.tmp/B/6cdb974236134267a7e4d6f4f42abaa2 as hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/697cd6a363e514bff0d4ec954fc4034b/B/6cdb974236134267a7e4d6f4f42abaa2 2024-11-16T10:00:09,624 DEBUG [RS:0;037c58a19ddb:37267-shortCompactions-0 {}] compactions.Compactor(225): Compacting 475bfdd4b2ed4ab9b2d86b58ceb856f3, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=284, earliestPutTs=1731751207241 2024-11-16T10:00:09,625 DEBUG [RS:0;037c58a19ddb:37267-shortCompactions-0 {}] compactions.Compactor(225): Compacting 1d64ef6ef6ef4156812f479b5da065f6, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=313, earliestPutTs=1731751207992 2024-11-16T10:00:09,628 INFO [RS:0;037c58a19ddb:37267-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 697cd6a363e514bff0d4ec954fc4034b/B of 697cd6a363e514bff0d4ec954fc4034b into 6cdb974236134267a7e4d6f4f42abaa2(size=12.5 K), total size for store is 12.5 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-16T10:00:09,628 DEBUG [RS:0;037c58a19ddb:37267-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 697cd6a363e514bff0d4ec954fc4034b: 2024-11-16T10:00:09,628 INFO [RS:0;037c58a19ddb:37267-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731751192181.697cd6a363e514bff0d4ec954fc4034b., storeName=697cd6a363e514bff0d4ec954fc4034b/B, priority=13, startTime=1731751209198; duration=0sec 2024-11-16T10:00:09,628 DEBUG [RS:0;037c58a19ddb:37267-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-16T10:00:09,628 DEBUG [RS:0;037c58a19ddb:37267-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 697cd6a363e514bff0d4ec954fc4034b:B 2024-11-16T10:00:09,632 INFO [RS:0;037c58a19ddb:37267-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 697cd6a363e514bff0d4ec954fc4034b#C#compaction#440 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-16T10:00:09,633 DEBUG [RS:0;037c58a19ddb:37267-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/697cd6a363e514bff0d4ec954fc4034b/.tmp/C/6846a39ce2564b2cbeec112bc41ff608 is 50, key is test_row_0/C:col10/1731751207992/Put/seqid=0 2024-11-16T10:00:09,636 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37591 is added to blk_1073742338_1514 (size=12847) 2024-11-16T10:00:09,677 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=697cd6a363e514bff0d4ec954fc4034b, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T10:00:09,677 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] ipc.CallRunner(138): callId: 71 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36086 deadline: 1731751269672, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=697cd6a363e514bff0d4ec954fc4034b, server=037c58a19ddb,37267,1731751077844 2024-11-16T10:00:09,678 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=697cd6a363e514bff0d4ec954fc4034b, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T10:00:09,678 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] ipc.CallRunner(138): callId: 125 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36012 deadline: 1731751269673, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=697cd6a363e514bff0d4ec954fc4034b, server=037c58a19ddb,37267,1731751077844 2024-11-16T10:00:09,678 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731751192181.697cd6a363e514bff0d4ec954fc4034b., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 , the old value is region=TestAcidGuarantees,,1731751192181.697cd6a363e514bff0d4ec954fc4034b., hostname=037c58a19ddb,37267,1731751077844, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=697cd6a363e514bff0d4ec954fc4034b, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:00:09,678 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731751192181.697cd6a363e514bff0d4ec954fc4034b., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 , the old value is region=TestAcidGuarantees,,1731751192181.697cd6a363e514bff0d4ec954fc4034b., hostname=037c58a19ddb,37267,1731751077844, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=697cd6a363e514bff0d4ec954fc4034b, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:00:09,678 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731751192181.697cd6a363e514bff0d4ec954fc4034b., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=697cd6a363e514bff0d4ec954fc4034b, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:00:09,678 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731751192181.697cd6a363e514bff0d4ec954fc4034b., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=697cd6a363e514bff0d4ec954fc4034b, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:00:09,678 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731751192181.697cd6a363e514bff0d4ec954fc4034b., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 because the exception is null or not the one we care about 2024-11-16T10:00:09,678 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731751192181.697cd6a363e514bff0d4ec954fc4034b., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 because the exception is null or not the one we care about 2024-11-16T10:00:09,688 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=697cd6a363e514bff0d4ec954fc4034b, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T10:00:09,688 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] ipc.CallRunner(138): callId: 134 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36058 deadline: 1731751269683, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=697cd6a363e514bff0d4ec954fc4034b, server=037c58a19ddb,37267,1731751077844 2024-11-16T10:00:09,689 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=697cd6a363e514bff0d4ec954fc4034b, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T10:00:09,689 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731751192181.697cd6a363e514bff0d4ec954fc4034b., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 , the old value is region=TestAcidGuarantees,,1731751192181.697cd6a363e514bff0d4ec954fc4034b., hostname=037c58a19ddb,37267,1731751077844, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=697cd6a363e514bff0d4ec954fc4034b, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:00:09,689 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] ipc.CallRunner(138): callId: 123 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36022 deadline: 1731751269683, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=697cd6a363e514bff0d4ec954fc4034b, server=037c58a19ddb,37267,1731751077844 2024-11-16T10:00:09,689 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731751192181.697cd6a363e514bff0d4ec954fc4034b., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=697cd6a363e514bff0d4ec954fc4034b, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:00:09,689 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731751192181.697cd6a363e514bff0d4ec954fc4034b., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 because the exception is null or not the one we care about 2024-11-16T10:00:09,689 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731751192181.697cd6a363e514bff0d4ec954fc4034b., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 , the old value is region=TestAcidGuarantees,,1731751192181.697cd6a363e514bff0d4ec954fc4034b., hostname=037c58a19ddb,37267,1731751077844, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=697cd6a363e514bff0d4ec954fc4034b, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:00:09,689 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731751192181.697cd6a363e514bff0d4ec954fc4034b., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=697cd6a363e514bff0d4ec954fc4034b, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:00:09,689 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731751192181.697cd6a363e514bff0d4ec954fc4034b., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 because the exception is null or not the one we care about 2024-11-16T10:00:09,875 INFO [RS_FLUSH_OPERATIONS-regionserver/037c58a19ddb:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=324 (bloomFilter=true), to=hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/697cd6a363e514bff0d4ec954fc4034b/.tmp/B/c5f885ecf9af4420be16cd8dd42d29cf 2024-11-16T10:00:09,882 DEBUG [RS_FLUSH_OPERATIONS-regionserver/037c58a19ddb:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/697cd6a363e514bff0d4ec954fc4034b/.tmp/C/9237610090d64e16b429d0a108aa1d65 is 50, key is test_row_0/C:col10/1731751208352/Put/seqid=0 2024-11-16T10:00:09,887 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37591 is added to blk_1073742339_1515 (size=12301) 2024-11-16T10:00:09,888 INFO [RS_FLUSH_OPERATIONS-regionserver/037c58a19ddb:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=324 (bloomFilter=true), to=hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/697cd6a363e514bff0d4ec954fc4034b/.tmp/C/9237610090d64e16b429d0a108aa1d65 2024-11-16T10:00:09,891 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=697cd6a363e514bff0d4ec954fc4034b, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T10:00:09,891 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] ipc.CallRunner(138): callId: 72 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36086 deadline: 1731751269882, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=697cd6a363e514bff0d4ec954fc4034b, server=037c58a19ddb,37267,1731751077844 2024-11-16T10:00:09,891 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=697cd6a363e514bff0d4ec954fc4034b, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T10:00:09,891 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] ipc.CallRunner(138): callId: 126 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36012 deadline: 1731751269883, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=697cd6a363e514bff0d4ec954fc4034b, server=037c58a19ddb,37267,1731751077844 2024-11-16T10:00:09,893 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731751192181.697cd6a363e514bff0d4ec954fc4034b., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 , the old value is region=TestAcidGuarantees,,1731751192181.697cd6a363e514bff0d4ec954fc4034b., hostname=037c58a19ddb,37267,1731751077844, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=697cd6a363e514bff0d4ec954fc4034b, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:00:09,893 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731751192181.697cd6a363e514bff0d4ec954fc4034b., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 , the old value is region=TestAcidGuarantees,,1731751192181.697cd6a363e514bff0d4ec954fc4034b., hostname=037c58a19ddb,37267,1731751077844, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=697cd6a363e514bff0d4ec954fc4034b, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:00:09,893 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731751192181.697cd6a363e514bff0d4ec954fc4034b., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=697cd6a363e514bff0d4ec954fc4034b, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:00:09,893 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731751192181.697cd6a363e514bff0d4ec954fc4034b., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=697cd6a363e514bff0d4ec954fc4034b, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:00:09,893 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731751192181.697cd6a363e514bff0d4ec954fc4034b., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 because the exception is null or not the one we care about 2024-11-16T10:00:09,893 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731751192181.697cd6a363e514bff0d4ec954fc4034b., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 because the exception is null or not the one we care about 2024-11-16T10:00:09,894 DEBUG [RS_FLUSH_OPERATIONS-regionserver/037c58a19ddb:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/697cd6a363e514bff0d4ec954fc4034b/.tmp/A/56afeb41771c4583b16aa0eb393faa0a as hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/697cd6a363e514bff0d4ec954fc4034b/A/56afeb41771c4583b16aa0eb393faa0a 2024-11-16T10:00:09,899 INFO [RS_FLUSH_OPERATIONS-regionserver/037c58a19ddb:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/697cd6a363e514bff0d4ec954fc4034b/A/56afeb41771c4583b16aa0eb393faa0a, entries=150, sequenceid=324, filesize=12.0 K 2024-11-16T10:00:09,900 DEBUG [RS_FLUSH_OPERATIONS-regionserver/037c58a19ddb:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/697cd6a363e514bff0d4ec954fc4034b/.tmp/B/c5f885ecf9af4420be16cd8dd42d29cf as hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/697cd6a363e514bff0d4ec954fc4034b/B/c5f885ecf9af4420be16cd8dd42d29cf 2024-11-16T10:00:09,903 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=697cd6a363e514bff0d4ec954fc4034b, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T10:00:09,903 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] ipc.CallRunner(138): callId: 135 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36058 deadline: 1731751269893, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=697cd6a363e514bff0d4ec954fc4034b, server=037c58a19ddb,37267,1731751077844 2024-11-16T10:00:09,903 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=697cd6a363e514bff0d4ec954fc4034b, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T10:00:09,903 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] ipc.CallRunner(138): callId: 124 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36022 deadline: 1731751269893, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=697cd6a363e514bff0d4ec954fc4034b, server=037c58a19ddb,37267,1731751077844 2024-11-16T10:00:09,903 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731751192181.697cd6a363e514bff0d4ec954fc4034b., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 , the old value is region=TestAcidGuarantees,,1731751192181.697cd6a363e514bff0d4ec954fc4034b., hostname=037c58a19ddb,37267,1731751077844, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=697cd6a363e514bff0d4ec954fc4034b, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:00:09,903 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731751192181.697cd6a363e514bff0d4ec954fc4034b., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=697cd6a363e514bff0d4ec954fc4034b, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:00:09,903 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731751192181.697cd6a363e514bff0d4ec954fc4034b., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 because the exception is null or not the one we care about 2024-11-16T10:00:09,903 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731751192181.697cd6a363e514bff0d4ec954fc4034b., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 , the old value is region=TestAcidGuarantees,,1731751192181.697cd6a363e514bff0d4ec954fc4034b., hostname=037c58a19ddb,37267,1731751077844, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=697cd6a363e514bff0d4ec954fc4034b, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:00:09,903 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731751192181.697cd6a363e514bff0d4ec954fc4034b., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=697cd6a363e514bff0d4ec954fc4034b, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:00:09,903 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731751192181.697cd6a363e514bff0d4ec954fc4034b., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 because the exception is null or not the one we care about 2024-11-16T10:00:09,905 INFO [RS_FLUSH_OPERATIONS-regionserver/037c58a19ddb:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/697cd6a363e514bff0d4ec954fc4034b/B/c5f885ecf9af4420be16cd8dd42d29cf, entries=150, sequenceid=324, filesize=12.0 K 2024-11-16T10:00:09,905 DEBUG [RS_FLUSH_OPERATIONS-regionserver/037c58a19ddb:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/697cd6a363e514bff0d4ec954fc4034b/.tmp/C/9237610090d64e16b429d0a108aa1d65 as hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/697cd6a363e514bff0d4ec954fc4034b/C/9237610090d64e16b429d0a108aa1d65 2024-11-16T10:00:09,909 INFO [RS_FLUSH_OPERATIONS-regionserver/037c58a19ddb:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/697cd6a363e514bff0d4ec954fc4034b/C/9237610090d64e16b429d0a108aa1d65, entries=150, sequenceid=324, filesize=12.0 K 2024-11-16T10:00:09,910 INFO [RS_FLUSH_OPERATIONS-regionserver/037c58a19ddb:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~141.33 KB/144720, currentSize=147.60 KB/151140 for 697cd6a363e514bff0d4ec954fc4034b in 470ms, sequenceid=324, compaction requested=false 2024-11-16T10:00:09,910 DEBUG [RS_FLUSH_OPERATIONS-regionserver/037c58a19ddb:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.HRegion(2603): Flush status journal for 697cd6a363e514bff0d4ec954fc4034b: 2024-11-16T10:00:09,910 DEBUG [RS_FLUSH_OPERATIONS-regionserver/037c58a19ddb:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731751192181.697cd6a363e514bff0d4ec954fc4034b. 2024-11-16T10:00:09,910 DEBUG [RS_FLUSH_OPERATIONS-regionserver/037c58a19ddb:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=132 2024-11-16T10:00:09,911 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37877 {}] master.HMaster(4169): Remote procedure done, pid=132 2024-11-16T10:00:09,912 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37877 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=131 2024-11-16T10:00:09,914 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=132, resume processing ppid=131 2024-11-16T10:00:09,914 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=132, ppid=131, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 625 msec 2024-11-16T10:00:09,916 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=131, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=131, table=TestAcidGuarantees in 631 msec 2024-11-16T10:00:10,041 DEBUG [RS:0;037c58a19ddb:37267-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/697cd6a363e514bff0d4ec954fc4034b/.tmp/C/6846a39ce2564b2cbeec112bc41ff608 as hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/697cd6a363e514bff0d4ec954fc4034b/C/6846a39ce2564b2cbeec112bc41ff608 2024-11-16T10:00:10,046 INFO [RS:0;037c58a19ddb:37267-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 697cd6a363e514bff0d4ec954fc4034b/C of 697cd6a363e514bff0d4ec954fc4034b into 6846a39ce2564b2cbeec112bc41ff608(size=12.5 K), total size for store is 24.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-16T10:00:10,046 DEBUG [RS:0;037c58a19ddb:37267-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 697cd6a363e514bff0d4ec954fc4034b: 2024-11-16T10:00:10,046 INFO [RS:0;037c58a19ddb:37267-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731751192181.697cd6a363e514bff0d4ec954fc4034b., storeName=697cd6a363e514bff0d4ec954fc4034b/C, priority=13, startTime=1731751209198; duration=0sec 2024-11-16T10:00:10,046 DEBUG [RS:0;037c58a19ddb:37267-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-16T10:00:10,046 DEBUG [RS:0;037c58a19ddb:37267-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 697cd6a363e514bff0d4ec954fc4034b:C 2024-11-16T10:00:10,206 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] regionserver.HRegion(8855): Flush requested on 697cd6a363e514bff0d4ec954fc4034b 2024-11-16T10:00:10,207 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-16T10:00:10,207 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 697cd6a363e514bff0d4ec954fc4034b 3/3 column families, dataSize=154.31 KB heapSize=405.05 KB 2024-11-16T10:00:10,213 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/697cd6a363e514bff0d4ec954fc4034b/.tmp/A/25d58730a74d47b9ae42efbe06a1c749 is 50, key is test_row_0/A:col10/1731751210205/Put/seqid=0 2024-11-16T10:00:10,219 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37591 is added to blk_1073742340_1516 (size=17181) 2024-11-16T10:00:10,231 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=697cd6a363e514bff0d4ec954fc4034b, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T10:00:10,232 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] ipc.CallRunner(138): callId: 137 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36058 deadline: 1731751270219, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=697cd6a363e514bff0d4ec954fc4034b, server=037c58a19ddb,37267,1731751077844 2024-11-16T10:00:10,232 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=697cd6a363e514bff0d4ec954fc4034b, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T10:00:10,232 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731751192181.697cd6a363e514bff0d4ec954fc4034b., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 , the old value is region=TestAcidGuarantees,,1731751192181.697cd6a363e514bff0d4ec954fc4034b., hostname=037c58a19ddb,37267,1731751077844, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=697cd6a363e514bff0d4ec954fc4034b, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:00:10,232 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] ipc.CallRunner(138): callId: 126 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36022 deadline: 1731751270221, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=697cd6a363e514bff0d4ec954fc4034b, server=037c58a19ddb,37267,1731751077844 2024-11-16T10:00:10,232 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731751192181.697cd6a363e514bff0d4ec954fc4034b., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=697cd6a363e514bff0d4ec954fc4034b, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:00:10,232 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731751192181.697cd6a363e514bff0d4ec954fc4034b., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 because the exception is null or not the one we care about 2024-11-16T10:00:10,232 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731751192181.697cd6a363e514bff0d4ec954fc4034b., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 , the old value is region=TestAcidGuarantees,,1731751192181.697cd6a363e514bff0d4ec954fc4034b., hostname=037c58a19ddb,37267,1731751077844, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=697cd6a363e514bff0d4ec954fc4034b, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:00:10,232 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731751192181.697cd6a363e514bff0d4ec954fc4034b., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=697cd6a363e514bff0d4ec954fc4034b, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:00:10,232 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731751192181.697cd6a363e514bff0d4ec954fc4034b., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 because the exception is null or not the one we care about 2024-11-16T10:00:10,232 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=697cd6a363e514bff0d4ec954fc4034b, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T10:00:10,232 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] ipc.CallRunner(138): callId: 130 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36012 deadline: 1731751270223, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=697cd6a363e514bff0d4ec954fc4034b, server=037c58a19ddb,37267,1731751077844 2024-11-16T10:00:10,233 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=697cd6a363e514bff0d4ec954fc4034b, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T10:00:10,233 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] ipc.CallRunner(138): callId: 76 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36086 deadline: 1731751270224, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=697cd6a363e514bff0d4ec954fc4034b, server=037c58a19ddb,37267,1731751077844 2024-11-16T10:00:10,233 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731751192181.697cd6a363e514bff0d4ec954fc4034b., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 , the old value is region=TestAcidGuarantees,,1731751192181.697cd6a363e514bff0d4ec954fc4034b., hostname=037c58a19ddb,37267,1731751077844, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=697cd6a363e514bff0d4ec954fc4034b, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:00:10,233 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731751192181.697cd6a363e514bff0d4ec954fc4034b., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=697cd6a363e514bff0d4ec954fc4034b, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:00:10,233 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731751192181.697cd6a363e514bff0d4ec954fc4034b., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 because the exception is null or not the one we care about 2024-11-16T10:00:10,233 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731751192181.697cd6a363e514bff0d4ec954fc4034b., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 , the old value is region=TestAcidGuarantees,,1731751192181.697cd6a363e514bff0d4ec954fc4034b., hostname=037c58a19ddb,37267,1731751077844, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=697cd6a363e514bff0d4ec954fc4034b, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:00:10,233 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731751192181.697cd6a363e514bff0d4ec954fc4034b., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=697cd6a363e514bff0d4ec954fc4034b, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:00:10,233 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731751192181.697cd6a363e514bff0d4ec954fc4034b., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 because the exception is null or not the one we care about 2024-11-16T10:00:10,349 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=697cd6a363e514bff0d4ec954fc4034b, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T10:00:10,350 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] ipc.CallRunner(138): callId: 138 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36058 deadline: 1731751270342, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=697cd6a363e514bff0d4ec954fc4034b, server=037c58a19ddb,37267,1731751077844 2024-11-16T10:00:10,350 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=697cd6a363e514bff0d4ec954fc4034b, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T10:00:10,350 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] ipc.CallRunner(138): callId: 127 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36022 deadline: 1731751270342, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=697cd6a363e514bff0d4ec954fc4034b, server=037c58a19ddb,37267,1731751077844 2024-11-16T10:00:10,350 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731751192181.697cd6a363e514bff0d4ec954fc4034b., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 , the old value is region=TestAcidGuarantees,,1731751192181.697cd6a363e514bff0d4ec954fc4034b., hostname=037c58a19ddb,37267,1731751077844, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=697cd6a363e514bff0d4ec954fc4034b, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:00:10,350 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731751192181.697cd6a363e514bff0d4ec954fc4034b., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=697cd6a363e514bff0d4ec954fc4034b, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:00:10,350 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731751192181.697cd6a363e514bff0d4ec954fc4034b., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 because the exception is null or not the one we care about 2024-11-16T10:00:10,350 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=697cd6a363e514bff0d4ec954fc4034b, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T10:00:10,350 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] ipc.CallRunner(138): callId: 131 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36012 deadline: 1731751270342, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=697cd6a363e514bff0d4ec954fc4034b, server=037c58a19ddb,37267,1731751077844 2024-11-16T10:00:10,350 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731751192181.697cd6a363e514bff0d4ec954fc4034b., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 , the old value is region=TestAcidGuarantees,,1731751192181.697cd6a363e514bff0d4ec954fc4034b., hostname=037c58a19ddb,37267,1731751077844, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=697cd6a363e514bff0d4ec954fc4034b, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:00:10,350 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731751192181.697cd6a363e514bff0d4ec954fc4034b., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=697cd6a363e514bff0d4ec954fc4034b, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:00:10,350 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731751192181.697cd6a363e514bff0d4ec954fc4034b., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 because the exception is null or not the one we care about 2024-11-16T10:00:10,350 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=697cd6a363e514bff0d4ec954fc4034b, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T10:00:10,350 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731751192181.697cd6a363e514bff0d4ec954fc4034b., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 , the old value is region=TestAcidGuarantees,,1731751192181.697cd6a363e514bff0d4ec954fc4034b., hostname=037c58a19ddb,37267,1731751077844, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=697cd6a363e514bff0d4ec954fc4034b, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:00:10,350 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] ipc.CallRunner(138): callId: 77 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36086 deadline: 1731751270343, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=697cd6a363e514bff0d4ec954fc4034b, server=037c58a19ddb,37267,1731751077844 2024-11-16T10:00:10,350 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731751192181.697cd6a363e514bff0d4ec954fc4034b., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=697cd6a363e514bff0d4ec954fc4034b, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:00:10,350 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731751192181.697cd6a363e514bff0d4ec954fc4034b., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 because the exception is null or not the one we care about 2024-11-16T10:00:10,351 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731751192181.697cd6a363e514bff0d4ec954fc4034b., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 , the old value is region=TestAcidGuarantees,,1731751192181.697cd6a363e514bff0d4ec954fc4034b., hostname=037c58a19ddb,37267,1731751077844, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=697cd6a363e514bff0d4ec954fc4034b, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:00:10,351 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731751192181.697cd6a363e514bff0d4ec954fc4034b., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=697cd6a363e514bff0d4ec954fc4034b, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:00:10,351 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731751192181.697cd6a363e514bff0d4ec954fc4034b., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 because the exception is null or not the one we care about 2024-11-16T10:00:10,422 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37877 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=131 2024-11-16T10:00:10,423 INFO [RPCClient-NioEventLoopGroup-4-10 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-16T10:00:10,423 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37877 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-16T10:00:10,424 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37877 {}] procedure2.ProcedureExecutor(1139): Stored pid=133, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=133, table=TestAcidGuarantees 2024-11-16T10:00:10,425 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37877 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=133 2024-11-16T10:00:10,425 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=133, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=133, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-16T10:00:10,426 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=133, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=133, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-16T10:00:10,426 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=134, ppid=133, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-16T10:00:10,533 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37877 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=133 2024-11-16T10:00:10,559 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=697cd6a363e514bff0d4ec954fc4034b, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T10:00:10,559 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] ipc.CallRunner(138): callId: 139 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36058 deadline: 1731751270552, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=697cd6a363e514bff0d4ec954fc4034b, server=037c58a19ddb,37267,1731751077844 2024-11-16T10:00:10,559 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=697cd6a363e514bff0d4ec954fc4034b, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T10:00:10,559 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731751192181.697cd6a363e514bff0d4ec954fc4034b., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 , the old value is region=TestAcidGuarantees,,1731751192181.697cd6a363e514bff0d4ec954fc4034b., hostname=037c58a19ddb,37267,1731751077844, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=697cd6a363e514bff0d4ec954fc4034b, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:00:10,559 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] ipc.CallRunner(138): callId: 128 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36022 deadline: 1731751270553, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=697cd6a363e514bff0d4ec954fc4034b, server=037c58a19ddb,37267,1731751077844 2024-11-16T10:00:10,559 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731751192181.697cd6a363e514bff0d4ec954fc4034b., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=697cd6a363e514bff0d4ec954fc4034b, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:00:10,559 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731751192181.697cd6a363e514bff0d4ec954fc4034b., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 because the exception is null or not the one we care about 2024-11-16T10:00:10,560 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=697cd6a363e514bff0d4ec954fc4034b, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T10:00:10,560 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731751192181.697cd6a363e514bff0d4ec954fc4034b., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 , the old value is region=TestAcidGuarantees,,1731751192181.697cd6a363e514bff0d4ec954fc4034b., hostname=037c58a19ddb,37267,1731751077844, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=697cd6a363e514bff0d4ec954fc4034b, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:00:10,560 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731751192181.697cd6a363e514bff0d4ec954fc4034b., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=697cd6a363e514bff0d4ec954fc4034b, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:00:10,560 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731751192181.697cd6a363e514bff0d4ec954fc4034b., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 because the exception is null or not the one we care about 2024-11-16T10:00:10,560 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] ipc.CallRunner(138): callId: 132 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36012 deadline: 1731751270553, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=697cd6a363e514bff0d4ec954fc4034b, server=037c58a19ddb,37267,1731751077844 2024-11-16T10:00:10,560 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=697cd6a363e514bff0d4ec954fc4034b, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T10:00:10,560 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731751192181.697cd6a363e514bff0d4ec954fc4034b., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 , the old value is region=TestAcidGuarantees,,1731751192181.697cd6a363e514bff0d4ec954fc4034b., hostname=037c58a19ddb,37267,1731751077844, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=697cd6a363e514bff0d4ec954fc4034b, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:00:10,560 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] ipc.CallRunner(138): callId: 78 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36086 deadline: 1731751270553, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=697cd6a363e514bff0d4ec954fc4034b, server=037c58a19ddb,37267,1731751077844 2024-11-16T10:00:10,560 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731751192181.697cd6a363e514bff0d4ec954fc4034b., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=697cd6a363e514bff0d4ec954fc4034b, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:00:10,560 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731751192181.697cd6a363e514bff0d4ec954fc4034b., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 because the exception is null or not the one we care about 2024-11-16T10:00:10,560 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731751192181.697cd6a363e514bff0d4ec954fc4034b., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 , the old value is region=TestAcidGuarantees,,1731751192181.697cd6a363e514bff0d4ec954fc4034b., hostname=037c58a19ddb,37267,1731751077844, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=697cd6a363e514bff0d4ec954fc4034b, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:00:10,560 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731751192181.697cd6a363e514bff0d4ec954fc4034b., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=697cd6a363e514bff0d4ec954fc4034b, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:00:10,560 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731751192181.697cd6a363e514bff0d4ec954fc4034b., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 because the exception is null or not the one we care about 2024-11-16T10:00:10,578 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=37267 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=134 2024-11-16T10:00:10,578 DEBUG [RS_FLUSH_OPERATIONS-regionserver/037c58a19ddb:0-2 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731751192181.697cd6a363e514bff0d4ec954fc4034b. 2024-11-16T10:00:10,578 DEBUG [RS_FLUSH_OPERATIONS-regionserver/037c58a19ddb:0-2 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731751192181.697cd6a363e514bff0d4ec954fc4034b. as already flushing 2024-11-16T10:00:10,578 DEBUG [RS_FLUSH_OPERATIONS-regionserver/037c58a19ddb:0-2 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731751192181.697cd6a363e514bff0d4ec954fc4034b. 2024-11-16T10:00:10,578 ERROR [RS_FLUSH_OPERATIONS-regionserver/037c58a19ddb:0-2 {event_type=RS_FLUSH_REGIONS, pid=134}] handler.RSProcedureHandler(58): pid=134 java.io.IOException: Unable to complete flush {ENCODED => 697cd6a363e514bff0d4ec954fc4034b, NAME => 'TestAcidGuarantees,,1731751192181.697cd6a363e514bff0d4ec954fc4034b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T10:00:10,578 DEBUG [RS_FLUSH_OPERATIONS-regionserver/037c58a19ddb:0-2 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=134 java.io.IOException: Unable to complete flush {ENCODED => 697cd6a363e514bff0d4ec954fc4034b, NAME => 'TestAcidGuarantees,,1731751192181.697cd6a363e514bff0d4ec954fc4034b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T10:00:10,579 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37877 {}] master.HMaster(4177): Remote procedure failed, pid=134 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 697cd6a363e514bff0d4ec954fc4034b, NAME => 'TestAcidGuarantees,,1731751192181.697cd6a363e514bff0d4ec954fc4034b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 697cd6a363e514bff0d4ec954fc4034b, NAME => 'TestAcidGuarantees,,1731751192181.697cd6a363e514bff0d4ec954fc4034b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T10:00:10,620 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=53.67 KB at sequenceid=354 (bloomFilter=true), to=hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/697cd6a363e514bff0d4ec954fc4034b/.tmp/A/25d58730a74d47b9ae42efbe06a1c749 2024-11-16T10:00:10,627 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/697cd6a363e514bff0d4ec954fc4034b/.tmp/B/afcabd057a004b74bc5fe9707dc6c2ca is 50, key is test_row_0/B:col10/1731751210205/Put/seqid=0 2024-11-16T10:00:10,630 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37591 is added to blk_1073742341_1517 (size=12301) 2024-11-16T10:00:10,630 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=53.67 KB at sequenceid=354 (bloomFilter=true), to=hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/697cd6a363e514bff0d4ec954fc4034b/.tmp/B/afcabd057a004b74bc5fe9707dc6c2ca 2024-11-16T10:00:10,637 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/697cd6a363e514bff0d4ec954fc4034b/.tmp/C/0a8b4857d6014068ba12683461bf5532 is 50, key is test_row_0/C:col10/1731751210205/Put/seqid=0 2024-11-16T10:00:10,640 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37591 is added to blk_1073742342_1518 (size=12301) 2024-11-16T10:00:10,731 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=37267 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=134 2024-11-16T10:00:10,731 DEBUG [RS_FLUSH_OPERATIONS-regionserver/037c58a19ddb:0-0 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731751192181.697cd6a363e514bff0d4ec954fc4034b. 2024-11-16T10:00:10,731 DEBUG [RS_FLUSH_OPERATIONS-regionserver/037c58a19ddb:0-0 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731751192181.697cd6a363e514bff0d4ec954fc4034b. as already flushing 2024-11-16T10:00:10,731 DEBUG [RS_FLUSH_OPERATIONS-regionserver/037c58a19ddb:0-0 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731751192181.697cd6a363e514bff0d4ec954fc4034b. 2024-11-16T10:00:10,732 ERROR [RS_FLUSH_OPERATIONS-regionserver/037c58a19ddb:0-0 {event_type=RS_FLUSH_REGIONS, pid=134}] handler.RSProcedureHandler(58): pid=134 java.io.IOException: Unable to complete flush {ENCODED => 697cd6a363e514bff0d4ec954fc4034b, NAME => 'TestAcidGuarantees,,1731751192181.697cd6a363e514bff0d4ec954fc4034b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T10:00:10,732 DEBUG [RS_FLUSH_OPERATIONS-regionserver/037c58a19ddb:0-0 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=134 java.io.IOException: Unable to complete flush {ENCODED => 697cd6a363e514bff0d4ec954fc4034b, NAME => 'TestAcidGuarantees,,1731751192181.697cd6a363e514bff0d4ec954fc4034b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T10:00:10,732 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37877 {}] master.HMaster(4177): Remote procedure failed, pid=134 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 697cd6a363e514bff0d4ec954fc4034b, NAME => 'TestAcidGuarantees,,1731751192181.697cd6a363e514bff0d4ec954fc4034b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 697cd6a363e514bff0d4ec954fc4034b, NAME => 'TestAcidGuarantees,,1731751192181.697cd6a363e514bff0d4ec954fc4034b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T10:00:10,743 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37877 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=133 2024-11-16T10:00:10,869 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=697cd6a363e514bff0d4ec954fc4034b, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T10:00:10,869 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] ipc.CallRunner(138): callId: 140 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36058 deadline: 1731751270863, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=697cd6a363e514bff0d4ec954fc4034b, server=037c58a19ddb,37267,1731751077844 2024-11-16T10:00:10,869 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=697cd6a363e514bff0d4ec954fc4034b, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T10:00:10,870 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731751192181.697cd6a363e514bff0d4ec954fc4034b., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 , the old value is region=TestAcidGuarantees,,1731751192181.697cd6a363e514bff0d4ec954fc4034b., hostname=037c58a19ddb,37267,1731751077844, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=697cd6a363e514bff0d4ec954fc4034b, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:00:10,870 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] ipc.CallRunner(138): callId: 133 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36012 deadline: 1731751270863, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=697cd6a363e514bff0d4ec954fc4034b, server=037c58a19ddb,37267,1731751077844 2024-11-16T10:00:10,870 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731751192181.697cd6a363e514bff0d4ec954fc4034b., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=697cd6a363e514bff0d4ec954fc4034b, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:00:10,870 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731751192181.697cd6a363e514bff0d4ec954fc4034b., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 because the exception is null or not the one we care about 2024-11-16T10:00:10,870 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731751192181.697cd6a363e514bff0d4ec954fc4034b., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 , the old value is region=TestAcidGuarantees,,1731751192181.697cd6a363e514bff0d4ec954fc4034b., hostname=037c58a19ddb,37267,1731751077844, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=697cd6a363e514bff0d4ec954fc4034b, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:00:10,870 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731751192181.697cd6a363e514bff0d4ec954fc4034b., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=697cd6a363e514bff0d4ec954fc4034b, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:00:10,870 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731751192181.697cd6a363e514bff0d4ec954fc4034b., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 because the exception is null or not the one we care about 2024-11-16T10:00:10,880 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=697cd6a363e514bff0d4ec954fc4034b, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T10:00:10,880 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] ipc.CallRunner(138): callId: 129 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36022 deadline: 1731751270872, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=697cd6a363e514bff0d4ec954fc4034b, server=037c58a19ddb,37267,1731751077844 2024-11-16T10:00:10,880 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=697cd6a363e514bff0d4ec954fc4034b, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T10:00:10,880 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] ipc.CallRunner(138): callId: 79 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36086 deadline: 1731751270873, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=697cd6a363e514bff0d4ec954fc4034b, server=037c58a19ddb,37267,1731751077844 2024-11-16T10:00:10,881 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731751192181.697cd6a363e514bff0d4ec954fc4034b., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 , the old value is region=TestAcidGuarantees,,1731751192181.697cd6a363e514bff0d4ec954fc4034b., hostname=037c58a19ddb,37267,1731751077844, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=697cd6a363e514bff0d4ec954fc4034b, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:00:10,881 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731751192181.697cd6a363e514bff0d4ec954fc4034b., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=697cd6a363e514bff0d4ec954fc4034b, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:00:10,881 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731751192181.697cd6a363e514bff0d4ec954fc4034b., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 because the exception is null or not the one we care about 2024-11-16T10:00:10,881 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731751192181.697cd6a363e514bff0d4ec954fc4034b., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 , the old value is region=TestAcidGuarantees,,1731751192181.697cd6a363e514bff0d4ec954fc4034b., hostname=037c58a19ddb,37267,1731751077844, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=697cd6a363e514bff0d4ec954fc4034b, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:00:10,881 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731751192181.697cd6a363e514bff0d4ec954fc4034b., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=697cd6a363e514bff0d4ec954fc4034b, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:00:10,881 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731751192181.697cd6a363e514bff0d4ec954fc4034b., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 because the exception is null or not the one we care about 2024-11-16T10:00:10,884 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=37267 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=134 2024-11-16T10:00:10,885 DEBUG [RS_FLUSH_OPERATIONS-regionserver/037c58a19ddb:0-1 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731751192181.697cd6a363e514bff0d4ec954fc4034b. 2024-11-16T10:00:10,885 DEBUG [RS_FLUSH_OPERATIONS-regionserver/037c58a19ddb:0-1 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731751192181.697cd6a363e514bff0d4ec954fc4034b. as already flushing 2024-11-16T10:00:10,885 DEBUG [RS_FLUSH_OPERATIONS-regionserver/037c58a19ddb:0-1 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731751192181.697cd6a363e514bff0d4ec954fc4034b. 2024-11-16T10:00:10,885 ERROR [RS_FLUSH_OPERATIONS-regionserver/037c58a19ddb:0-1 {event_type=RS_FLUSH_REGIONS, pid=134}] handler.RSProcedureHandler(58): pid=134 java.io.IOException: Unable to complete flush {ENCODED => 697cd6a363e514bff0d4ec954fc4034b, NAME => 'TestAcidGuarantees,,1731751192181.697cd6a363e514bff0d4ec954fc4034b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T10:00:10,885 DEBUG [RS_FLUSH_OPERATIONS-regionserver/037c58a19ddb:0-1 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=134 java.io.IOException: Unable to complete flush {ENCODED => 697cd6a363e514bff0d4ec954fc4034b, NAME => 'TestAcidGuarantees,,1731751192181.697cd6a363e514bff0d4ec954fc4034b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T10:00:10,885 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37877 {}] master.HMaster(4177): Remote procedure failed, pid=134 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 697cd6a363e514bff0d4ec954fc4034b, NAME => 'TestAcidGuarantees,,1731751192181.697cd6a363e514bff0d4ec954fc4034b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 697cd6a363e514bff0d4ec954fc4034b, NAME => 'TestAcidGuarantees,,1731751192181.697cd6a363e514bff0d4ec954fc4034b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T10:00:11,036 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=37267 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=134 2024-11-16T10:00:11,037 DEBUG [RS_FLUSH_OPERATIONS-regionserver/037c58a19ddb:0-2 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731751192181.697cd6a363e514bff0d4ec954fc4034b. 2024-11-16T10:00:11,037 DEBUG [RS_FLUSH_OPERATIONS-regionserver/037c58a19ddb:0-2 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731751192181.697cd6a363e514bff0d4ec954fc4034b. as already flushing 2024-11-16T10:00:11,037 DEBUG [RS_FLUSH_OPERATIONS-regionserver/037c58a19ddb:0-2 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731751192181.697cd6a363e514bff0d4ec954fc4034b. 2024-11-16T10:00:11,037 ERROR [RS_FLUSH_OPERATIONS-regionserver/037c58a19ddb:0-2 {event_type=RS_FLUSH_REGIONS, pid=134}] handler.RSProcedureHandler(58): pid=134 java.io.IOException: Unable to complete flush {ENCODED => 697cd6a363e514bff0d4ec954fc4034b, NAME => 'TestAcidGuarantees,,1731751192181.697cd6a363e514bff0d4ec954fc4034b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T10:00:11,037 DEBUG [RS_FLUSH_OPERATIONS-regionserver/037c58a19ddb:0-2 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=134 java.io.IOException: Unable to complete flush {ENCODED => 697cd6a363e514bff0d4ec954fc4034b, NAME => 'TestAcidGuarantees,,1731751192181.697cd6a363e514bff0d4ec954fc4034b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T10:00:11,038 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37877 {}] master.HMaster(4177): Remote procedure failed, pid=134 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 697cd6a363e514bff0d4ec954fc4034b, NAME => 'TestAcidGuarantees,,1731751192181.697cd6a363e514bff0d4ec954fc4034b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 697cd6a363e514bff0d4ec954fc4034b, NAME => 'TestAcidGuarantees,,1731751192181.697cd6a363e514bff0d4ec954fc4034b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T10:00:11,040 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=53.67 KB at sequenceid=354 (bloomFilter=true), to=hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/697cd6a363e514bff0d4ec954fc4034b/.tmp/C/0a8b4857d6014068ba12683461bf5532 2024-11-16T10:00:11,044 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/697cd6a363e514bff0d4ec954fc4034b/.tmp/A/25d58730a74d47b9ae42efbe06a1c749 as hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/697cd6a363e514bff0d4ec954fc4034b/A/25d58730a74d47b9ae42efbe06a1c749 2024-11-16T10:00:11,047 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/697cd6a363e514bff0d4ec954fc4034b/A/25d58730a74d47b9ae42efbe06a1c749, entries=250, sequenceid=354, filesize=16.8 K 2024-11-16T10:00:11,048 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/697cd6a363e514bff0d4ec954fc4034b/.tmp/B/afcabd057a004b74bc5fe9707dc6c2ca as hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/697cd6a363e514bff0d4ec954fc4034b/B/afcabd057a004b74bc5fe9707dc6c2ca 2024-11-16T10:00:11,051 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/697cd6a363e514bff0d4ec954fc4034b/B/afcabd057a004b74bc5fe9707dc6c2ca, entries=150, sequenceid=354, filesize=12.0 K 2024-11-16T10:00:11,052 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/697cd6a363e514bff0d4ec954fc4034b/.tmp/C/0a8b4857d6014068ba12683461bf5532 as hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/697cd6a363e514bff0d4ec954fc4034b/C/0a8b4857d6014068ba12683461bf5532 2024-11-16T10:00:11,053 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37877 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=133 2024-11-16T10:00:11,055 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/697cd6a363e514bff0d4ec954fc4034b/C/0a8b4857d6014068ba12683461bf5532, entries=150, sequenceid=354, filesize=12.0 K 2024-11-16T10:00:11,056 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~161.02 KB/164880, heapSize ~422.58 KB/432720, currentSize=40.25 KB/41220 for 697cd6a363e514bff0d4ec954fc4034b in 849ms, sequenceid=354, compaction requested=true 2024-11-16T10:00:11,056 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 697cd6a363e514bff0d4ec954fc4034b: 2024-11-16T10:00:11,056 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 697cd6a363e514bff0d4ec954fc4034b:A, priority=-2147483648, current under compaction store size is 1 2024-11-16T10:00:11,056 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-16T10:00:11,056 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 697cd6a363e514bff0d4ec954fc4034b:B, priority=-2147483648, current under compaction store size is 2 2024-11-16T10:00:11,056 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-16T10:00:11,056 DEBUG [RS:0;037c58a19ddb:37267-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-16T10:00:11,056 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 697cd6a363e514bff0d4ec954fc4034b:C, priority=-2147483648, current under compaction store size is 3 2024-11-16T10:00:11,056 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-16T10:00:11,056 DEBUG [RS:0;037c58a19ddb:37267-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-16T10:00:11,057 DEBUG [RS:0;037c58a19ddb:37267-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 42329 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-16T10:00:11,057 DEBUG [RS:0;037c58a19ddb:37267-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37449 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-16T10:00:11,057 DEBUG [RS:0;037c58a19ddb:37267-longCompactions-0 {}] regionserver.HStore(1541): 697cd6a363e514bff0d4ec954fc4034b/B is initiating minor compaction (all files) 2024-11-16T10:00:11,057 DEBUG [RS:0;037c58a19ddb:37267-shortCompactions-0 {}] regionserver.HStore(1541): 697cd6a363e514bff0d4ec954fc4034b/A is initiating minor compaction (all files) 2024-11-16T10:00:11,058 INFO [RS:0;037c58a19ddb:37267-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 697cd6a363e514bff0d4ec954fc4034b/B in TestAcidGuarantees,,1731751192181.697cd6a363e514bff0d4ec954fc4034b. 2024-11-16T10:00:11,058 INFO [RS:0;037c58a19ddb:37267-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 697cd6a363e514bff0d4ec954fc4034b/A in TestAcidGuarantees,,1731751192181.697cd6a363e514bff0d4ec954fc4034b. 2024-11-16T10:00:11,058 INFO [RS:0;037c58a19ddb:37267-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/697cd6a363e514bff0d4ec954fc4034b/B/6cdb974236134267a7e4d6f4f42abaa2, hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/697cd6a363e514bff0d4ec954fc4034b/B/c5f885ecf9af4420be16cd8dd42d29cf, hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/697cd6a363e514bff0d4ec954fc4034b/B/afcabd057a004b74bc5fe9707dc6c2ca] into tmpdir=hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/697cd6a363e514bff0d4ec954fc4034b/.tmp, totalSize=36.6 K 2024-11-16T10:00:11,058 INFO [RS:0;037c58a19ddb:37267-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/697cd6a363e514bff0d4ec954fc4034b/A/c3c82a2503ed413f9db2739f818730bf, hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/697cd6a363e514bff0d4ec954fc4034b/A/56afeb41771c4583b16aa0eb393faa0a, hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/697cd6a363e514bff0d4ec954fc4034b/A/25d58730a74d47b9ae42efbe06a1c749] into tmpdir=hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/697cd6a363e514bff0d4ec954fc4034b/.tmp, totalSize=41.3 K 2024-11-16T10:00:11,058 DEBUG [RS:0;037c58a19ddb:37267-shortCompactions-0 {}] compactions.Compactor(225): Compacting c3c82a2503ed413f9db2739f818730bf, keycount=150, bloomtype=ROW, size=12.5 K, encoding=NONE, compression=NONE, seqNum=313, earliestPutTs=1731751207992 2024-11-16T10:00:11,058 DEBUG [RS:0;037c58a19ddb:37267-longCompactions-0 {}] compactions.Compactor(225): Compacting 6cdb974236134267a7e4d6f4f42abaa2, keycount=150, bloomtype=ROW, size=12.5 K, encoding=NONE, compression=NONE, seqNum=313, earliestPutTs=1731751207992 2024-11-16T10:00:11,058 DEBUG [RS:0;037c58a19ddb:37267-shortCompactions-0 {}] compactions.Compactor(225): Compacting 56afeb41771c4583b16aa0eb393faa0a, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=324, earliestPutTs=1731751208352 2024-11-16T10:00:11,058 DEBUG [RS:0;037c58a19ddb:37267-longCompactions-0 {}] compactions.Compactor(225): Compacting c5f885ecf9af4420be16cd8dd42d29cf, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=324, earliestPutTs=1731751208352 2024-11-16T10:00:11,059 DEBUG [RS:0;037c58a19ddb:37267-longCompactions-0 {}] compactions.Compactor(225): Compacting afcabd057a004b74bc5fe9707dc6c2ca, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=354, earliestPutTs=1731751209560 2024-11-16T10:00:11,059 DEBUG [RS:0;037c58a19ddb:37267-shortCompactions-0 {}] compactions.Compactor(225): Compacting 25d58730a74d47b9ae42efbe06a1c749, keycount=250, bloomtype=ROW, size=16.8 K, encoding=NONE, compression=NONE, seqNum=354, earliestPutTs=1731751209556 2024-11-16T10:00:11,065 INFO [RS:0;037c58a19ddb:37267-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 697cd6a363e514bff0d4ec954fc4034b#B#compaction#445 average throughput is unlimited, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-16T10:00:11,066 DEBUG [RS:0;037c58a19ddb:37267-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/697cd6a363e514bff0d4ec954fc4034b/.tmp/B/376083493aa8414fbc47f4cca9482837 is 50, key is test_row_0/B:col10/1731751210205/Put/seqid=0 2024-11-16T10:00:11,066 INFO [RS:0;037c58a19ddb:37267-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 697cd6a363e514bff0d4ec954fc4034b#A#compaction#446 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-16T10:00:11,067 DEBUG [RS:0;037c58a19ddb:37267-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/697cd6a363e514bff0d4ec954fc4034b/.tmp/A/6287c88120694b81afbdd7d41cdc78ff is 50, key is test_row_0/A:col10/1731751210205/Put/seqid=0 2024-11-16T10:00:11,070 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37591 is added to blk_1073742343_1519 (size=12949) 2024-11-16T10:00:11,074 DEBUG [RS:0;037c58a19ddb:37267-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/697cd6a363e514bff0d4ec954fc4034b/.tmp/B/376083493aa8414fbc47f4cca9482837 as hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/697cd6a363e514bff0d4ec954fc4034b/B/376083493aa8414fbc47f4cca9482837 2024-11-16T10:00:11,079 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37591 is added to blk_1073742344_1520 (size=12949) 2024-11-16T10:00:11,079 INFO [RS:0;037c58a19ddb:37267-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 697cd6a363e514bff0d4ec954fc4034b/B of 697cd6a363e514bff0d4ec954fc4034b into 376083493aa8414fbc47f4cca9482837(size=12.6 K), total size for store is 12.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-16T10:00:11,079 DEBUG [RS:0;037c58a19ddb:37267-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 697cd6a363e514bff0d4ec954fc4034b: 2024-11-16T10:00:11,079 INFO [RS:0;037c58a19ddb:37267-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731751192181.697cd6a363e514bff0d4ec954fc4034b., storeName=697cd6a363e514bff0d4ec954fc4034b/B, priority=13, startTime=1731751211056; duration=0sec 2024-11-16T10:00:11,079 DEBUG [RS:0;037c58a19ddb:37267-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-16T10:00:11,079 DEBUG [RS:0;037c58a19ddb:37267-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 697cd6a363e514bff0d4ec954fc4034b:B 2024-11-16T10:00:11,080 DEBUG [RS:0;037c58a19ddb:37267-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-16T10:00:11,080 DEBUG [RS:0;037c58a19ddb:37267-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37449 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-16T10:00:11,080 DEBUG [RS:0;037c58a19ddb:37267-longCompactions-0 {}] regionserver.HStore(1541): 697cd6a363e514bff0d4ec954fc4034b/C is initiating minor compaction (all files) 2024-11-16T10:00:11,080 INFO [RS:0;037c58a19ddb:37267-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 697cd6a363e514bff0d4ec954fc4034b/C in TestAcidGuarantees,,1731751192181.697cd6a363e514bff0d4ec954fc4034b. 2024-11-16T10:00:11,081 INFO [RS:0;037c58a19ddb:37267-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/697cd6a363e514bff0d4ec954fc4034b/C/6846a39ce2564b2cbeec112bc41ff608, hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/697cd6a363e514bff0d4ec954fc4034b/C/9237610090d64e16b429d0a108aa1d65, hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/697cd6a363e514bff0d4ec954fc4034b/C/0a8b4857d6014068ba12683461bf5532] into tmpdir=hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/697cd6a363e514bff0d4ec954fc4034b/.tmp, totalSize=36.6 K 2024-11-16T10:00:11,081 DEBUG [RS:0;037c58a19ddb:37267-longCompactions-0 {}] compactions.Compactor(225): Compacting 6846a39ce2564b2cbeec112bc41ff608, keycount=150, bloomtype=ROW, size=12.5 K, encoding=NONE, compression=NONE, seqNum=313, earliestPutTs=1731751207992 2024-11-16T10:00:11,081 DEBUG [RS:0;037c58a19ddb:37267-longCompactions-0 {}] compactions.Compactor(225): Compacting 9237610090d64e16b429d0a108aa1d65, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=324, earliestPutTs=1731751208352 2024-11-16T10:00:11,082 DEBUG [RS:0;037c58a19ddb:37267-longCompactions-0 {}] compactions.Compactor(225): Compacting 0a8b4857d6014068ba12683461bf5532, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=354, earliestPutTs=1731751209560 2024-11-16T10:00:11,088 INFO [RS:0;037c58a19ddb:37267-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 697cd6a363e514bff0d4ec954fc4034b#C#compaction#447 average throughput is unlimited, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-16T10:00:11,088 DEBUG [RS:0;037c58a19ddb:37267-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/697cd6a363e514bff0d4ec954fc4034b/.tmp/C/0c90ee06996748789e1d25093a7f1e25 is 50, key is test_row_0/C:col10/1731751210205/Put/seqid=0 2024-11-16T10:00:11,100 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37591 is added to blk_1073742345_1521 (size=12949) 2024-11-16T10:00:11,113 DEBUG [RS:0;037c58a19ddb:37267-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/697cd6a363e514bff0d4ec954fc4034b/.tmp/C/0c90ee06996748789e1d25093a7f1e25 as hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/697cd6a363e514bff0d4ec954fc4034b/C/0c90ee06996748789e1d25093a7f1e25 2024-11-16T10:00:11,117 INFO [RS:0;037c58a19ddb:37267-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 697cd6a363e514bff0d4ec954fc4034b/C of 697cd6a363e514bff0d4ec954fc4034b into 0c90ee06996748789e1d25093a7f1e25(size=12.6 K), total size for store is 12.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-16T10:00:11,117 DEBUG [RS:0;037c58a19ddb:37267-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 697cd6a363e514bff0d4ec954fc4034b: 2024-11-16T10:00:11,117 INFO [RS:0;037c58a19ddb:37267-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731751192181.697cd6a363e514bff0d4ec954fc4034b., storeName=697cd6a363e514bff0d4ec954fc4034b/C, priority=13, startTime=1731751211056; duration=0sec 2024-11-16T10:00:11,117 DEBUG [RS:0;037c58a19ddb:37267-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-16T10:00:11,117 DEBUG [RS:0;037c58a19ddb:37267-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 697cd6a363e514bff0d4ec954fc4034b:C 2024-11-16T10:00:11,190 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=37267 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=134 2024-11-16T10:00:11,190 DEBUG [RS_FLUSH_OPERATIONS-regionserver/037c58a19ddb:0-0 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731751192181.697cd6a363e514bff0d4ec954fc4034b. 2024-11-16T10:00:11,190 INFO [RS_FLUSH_OPERATIONS-regionserver/037c58a19ddb:0-0 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.HRegion(2902): Flushing 697cd6a363e514bff0d4ec954fc4034b 3/3 column families, dataSize=40.25 KB heapSize=106.22 KB 2024-11-16T10:00:11,194 DEBUG [RS_FLUSH_OPERATIONS-regionserver/037c58a19ddb:0-0 {event_type=RS_FLUSH_REGIONS, pid=134}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/697cd6a363e514bff0d4ec954fc4034b/.tmp/A/cdb34818a22c4599969803da6533b889 is 50, key is test_row_1/A:col10/1731751210222/Put/seqid=0 2024-11-16T10:00:11,199 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37591 is added to blk_1073742346_1522 (size=9857) 2024-11-16T10:00:11,387 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] regionserver.HRegion(8855): Flush requested on 697cd6a363e514bff0d4ec954fc4034b 2024-11-16T10:00:11,387 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731751192181.697cd6a363e514bff0d4ec954fc4034b. as already flushing 2024-11-16T10:00:11,441 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=697cd6a363e514bff0d4ec954fc4034b, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T10:00:11,441 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] ipc.CallRunner(138): callId: 148 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36058 deadline: 1731751271437, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=697cd6a363e514bff0d4ec954fc4034b, server=037c58a19ddb,37267,1731751077844 2024-11-16T10:00:11,442 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731751192181.697cd6a363e514bff0d4ec954fc4034b., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 , the old value is region=TestAcidGuarantees,,1731751192181.697cd6a363e514bff0d4ec954fc4034b., hostname=037c58a19ddb,37267,1731751077844, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=697cd6a363e514bff0d4ec954fc4034b, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:00:11,442 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731751192181.697cd6a363e514bff0d4ec954fc4034b., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=697cd6a363e514bff0d4ec954fc4034b, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:00:11,442 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731751192181.697cd6a363e514bff0d4ec954fc4034b., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 because the exception is null or not the one we care about 2024-11-16T10:00:11,449 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=697cd6a363e514bff0d4ec954fc4034b, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T10:00:11,449 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] ipc.CallRunner(138): callId: 135 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36022 deadline: 1731751271438, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=697cd6a363e514bff0d4ec954fc4034b, server=037c58a19ddb,37267,1731751077844 2024-11-16T10:00:11,449 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=697cd6a363e514bff0d4ec954fc4034b, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T10:00:11,449 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731751192181.697cd6a363e514bff0d4ec954fc4034b., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 , the old value is region=TestAcidGuarantees,,1731751192181.697cd6a363e514bff0d4ec954fc4034b., hostname=037c58a19ddb,37267,1731751077844, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=697cd6a363e514bff0d4ec954fc4034b, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:00:11,449 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=697cd6a363e514bff0d4ec954fc4034b, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T10:00:11,449 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731751192181.697cd6a363e514bff0d4ec954fc4034b., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=697cd6a363e514bff0d4ec954fc4034b, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:00:11,449 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] ipc.CallRunner(138): callId: 85 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36086 deadline: 1731751271439, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=697cd6a363e514bff0d4ec954fc4034b, server=037c58a19ddb,37267,1731751077844 2024-11-16T10:00:11,449 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731751192181.697cd6a363e514bff0d4ec954fc4034b., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 because the exception is null or not the one we care about 2024-11-16T10:00:11,449 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] ipc.CallRunner(138): callId: 141 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36012 deadline: 1731751271440, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=697cd6a363e514bff0d4ec954fc4034b, server=037c58a19ddb,37267,1731751077844 2024-11-16T10:00:11,449 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731751192181.697cd6a363e514bff0d4ec954fc4034b., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 , the old value is region=TestAcidGuarantees,,1731751192181.697cd6a363e514bff0d4ec954fc4034b., hostname=037c58a19ddb,37267,1731751077844, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=697cd6a363e514bff0d4ec954fc4034b, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:00:11,449 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731751192181.697cd6a363e514bff0d4ec954fc4034b., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=697cd6a363e514bff0d4ec954fc4034b, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:00:11,449 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731751192181.697cd6a363e514bff0d4ec954fc4034b., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 , the old value is region=TestAcidGuarantees,,1731751192181.697cd6a363e514bff0d4ec954fc4034b., hostname=037c58a19ddb,37267,1731751077844, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=697cd6a363e514bff0d4ec954fc4034b, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:00:11,449 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731751192181.697cd6a363e514bff0d4ec954fc4034b., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 because the exception is null or not the one we care about 2024-11-16T10:00:11,449 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731751192181.697cd6a363e514bff0d4ec954fc4034b., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=697cd6a363e514bff0d4ec954fc4034b, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:00:11,449 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731751192181.697cd6a363e514bff0d4ec954fc4034b., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 because the exception is null or not the one we care about 2024-11-16T10:00:11,483 DEBUG [RS:0;037c58a19ddb:37267-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/697cd6a363e514bff0d4ec954fc4034b/.tmp/A/6287c88120694b81afbdd7d41cdc78ff as hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/697cd6a363e514bff0d4ec954fc4034b/A/6287c88120694b81afbdd7d41cdc78ff 2024-11-16T10:00:11,486 INFO [RS:0;037c58a19ddb:37267-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 697cd6a363e514bff0d4ec954fc4034b/A of 697cd6a363e514bff0d4ec954fc4034b into 6287c88120694b81afbdd7d41cdc78ff(size=12.6 K), total size for store is 12.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-16T10:00:11,486 DEBUG [RS:0;037c58a19ddb:37267-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 697cd6a363e514bff0d4ec954fc4034b: 2024-11-16T10:00:11,486 INFO [RS:0;037c58a19ddb:37267-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731751192181.697cd6a363e514bff0d4ec954fc4034b., storeName=697cd6a363e514bff0d4ec954fc4034b/A, priority=13, startTime=1731751211056; duration=0sec 2024-11-16T10:00:11,487 DEBUG [RS:0;037c58a19ddb:37267-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-16T10:00:11,487 DEBUG [RS:0;037c58a19ddb:37267-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 697cd6a363e514bff0d4ec954fc4034b:A 2024-11-16T10:00:11,557 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=697cd6a363e514bff0d4ec954fc4034b, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T10:00:11,557 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] ipc.CallRunner(138): callId: 149 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36058 deadline: 1731751271552, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=697cd6a363e514bff0d4ec954fc4034b, server=037c58a19ddb,37267,1731751077844 2024-11-16T10:00:11,557 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=697cd6a363e514bff0d4ec954fc4034b, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T10:00:11,557 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731751192181.697cd6a363e514bff0d4ec954fc4034b., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 , the old value is region=TestAcidGuarantees,,1731751192181.697cd6a363e514bff0d4ec954fc4034b., hostname=037c58a19ddb,37267,1731751077844, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=697cd6a363e514bff0d4ec954fc4034b, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:00:11,557 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] ipc.CallRunner(138): callId: 136 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36022 deadline: 1731751271552, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=697cd6a363e514bff0d4ec954fc4034b, server=037c58a19ddb,37267,1731751077844 2024-11-16T10:00:11,557 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731751192181.697cd6a363e514bff0d4ec954fc4034b., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=697cd6a363e514bff0d4ec954fc4034b, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:00:11,557 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731751192181.697cd6a363e514bff0d4ec954fc4034b., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 because the exception is null or not the one we care about 2024-11-16T10:00:11,557 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=697cd6a363e514bff0d4ec954fc4034b, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T10:00:11,557 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731751192181.697cd6a363e514bff0d4ec954fc4034b., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 , the old value is region=TestAcidGuarantees,,1731751192181.697cd6a363e514bff0d4ec954fc4034b., hostname=037c58a19ddb,37267,1731751077844, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=697cd6a363e514bff0d4ec954fc4034b, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:00:11,558 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731751192181.697cd6a363e514bff0d4ec954fc4034b., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=697cd6a363e514bff0d4ec954fc4034b, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:00:11,558 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731751192181.697cd6a363e514bff0d4ec954fc4034b., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 because the exception is null or not the one we care about 2024-11-16T10:00:11,558 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] ipc.CallRunner(138): callId: 86 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36086 deadline: 1731751271553, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=697cd6a363e514bff0d4ec954fc4034b, server=037c58a19ddb,37267,1731751077844 2024-11-16T10:00:11,558 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=697cd6a363e514bff0d4ec954fc4034b, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T10:00:11,558 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731751192181.697cd6a363e514bff0d4ec954fc4034b., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 , the old value is region=TestAcidGuarantees,,1731751192181.697cd6a363e514bff0d4ec954fc4034b., hostname=037c58a19ddb,37267,1731751077844, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=697cd6a363e514bff0d4ec954fc4034b, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:00:11,558 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] ipc.CallRunner(138): callId: 142 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36012 deadline: 1731751271553, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=697cd6a363e514bff0d4ec954fc4034b, server=037c58a19ddb,37267,1731751077844 2024-11-16T10:00:11,558 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731751192181.697cd6a363e514bff0d4ec954fc4034b., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=697cd6a363e514bff0d4ec954fc4034b, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:00:11,558 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731751192181.697cd6a363e514bff0d4ec954fc4034b., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 because the exception is null or not the one we care about 2024-11-16T10:00:11,558 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731751192181.697cd6a363e514bff0d4ec954fc4034b., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 , the old value is region=TestAcidGuarantees,,1731751192181.697cd6a363e514bff0d4ec954fc4034b., hostname=037c58a19ddb,37267,1731751077844, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=697cd6a363e514bff0d4ec954fc4034b, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:00:11,558 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731751192181.697cd6a363e514bff0d4ec954fc4034b., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=697cd6a363e514bff0d4ec954fc4034b, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:00:11,558 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731751192181.697cd6a363e514bff0d4ec954fc4034b., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 because the exception is null or not the one we care about 2024-11-16T10:00:11,563 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37877 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=133 2024-11-16T10:00:11,599 INFO [RS_FLUSH_OPERATIONS-regionserver/037c58a19ddb:0-0 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=13.42 KB at sequenceid=365 (bloomFilter=true), to=hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/697cd6a363e514bff0d4ec954fc4034b/.tmp/A/cdb34818a22c4599969803da6533b889 2024-11-16T10:00:11,605 DEBUG [RS_FLUSH_OPERATIONS-regionserver/037c58a19ddb:0-0 {event_type=RS_FLUSH_REGIONS, pid=134}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/697cd6a363e514bff0d4ec954fc4034b/.tmp/B/743c3e50c2dc453da9f17314fa21a38a is 50, key is test_row_1/B:col10/1731751210222/Put/seqid=0 2024-11-16T10:00:11,608 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37591 is added to blk_1073742347_1523 (size=9857) 2024-11-16T10:00:11,769 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=697cd6a363e514bff0d4ec954fc4034b, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T10:00:11,769 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] ipc.CallRunner(138): callId: 150 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36058 deadline: 1731751271762, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=697cd6a363e514bff0d4ec954fc4034b, server=037c58a19ddb,37267,1731751077844 2024-11-16T10:00:11,769 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=697cd6a363e514bff0d4ec954fc4034b, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T10:00:11,769 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731751192181.697cd6a363e514bff0d4ec954fc4034b., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 , the old value is region=TestAcidGuarantees,,1731751192181.697cd6a363e514bff0d4ec954fc4034b., hostname=037c58a19ddb,37267,1731751077844, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=697cd6a363e514bff0d4ec954fc4034b, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:00:11,769 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] ipc.CallRunner(138): callId: 137 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36022 deadline: 1731751271762, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=697cd6a363e514bff0d4ec954fc4034b, server=037c58a19ddb,37267,1731751077844 2024-11-16T10:00:11,769 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731751192181.697cd6a363e514bff0d4ec954fc4034b., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=697cd6a363e514bff0d4ec954fc4034b, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:00:11,769 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731751192181.697cd6a363e514bff0d4ec954fc4034b., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 because the exception is null or not the one we care about 2024-11-16T10:00:11,769 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=697cd6a363e514bff0d4ec954fc4034b, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T10:00:11,770 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731751192181.697cd6a363e514bff0d4ec954fc4034b., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 , the old value is region=TestAcidGuarantees,,1731751192181.697cd6a363e514bff0d4ec954fc4034b., hostname=037c58a19ddb,37267,1731751077844, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=697cd6a363e514bff0d4ec954fc4034b, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:00:11,770 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731751192181.697cd6a363e514bff0d4ec954fc4034b., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=697cd6a363e514bff0d4ec954fc4034b, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:00:11,770 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] ipc.CallRunner(138): callId: 87 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36086 deadline: 1731751271763, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=697cd6a363e514bff0d4ec954fc4034b, server=037c58a19ddb,37267,1731751077844 2024-11-16T10:00:11,770 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731751192181.697cd6a363e514bff0d4ec954fc4034b., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 because the exception is null or not the one we care about 2024-11-16T10:00:11,770 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731751192181.697cd6a363e514bff0d4ec954fc4034b., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 , the old value is region=TestAcidGuarantees,,1731751192181.697cd6a363e514bff0d4ec954fc4034b., hostname=037c58a19ddb,37267,1731751077844, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=697cd6a363e514bff0d4ec954fc4034b, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:00:11,770 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=697cd6a363e514bff0d4ec954fc4034b, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T10:00:11,770 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731751192181.697cd6a363e514bff0d4ec954fc4034b., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=697cd6a363e514bff0d4ec954fc4034b, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:00:11,770 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731751192181.697cd6a363e514bff0d4ec954fc4034b., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 because the exception is null or not the one we care about 2024-11-16T10:00:11,770 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] ipc.CallRunner(138): callId: 143 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36012 deadline: 1731751271763, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=697cd6a363e514bff0d4ec954fc4034b, server=037c58a19ddb,37267,1731751077844 2024-11-16T10:00:11,770 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731751192181.697cd6a363e514bff0d4ec954fc4034b., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 , the old value is region=TestAcidGuarantees,,1731751192181.697cd6a363e514bff0d4ec954fc4034b., hostname=037c58a19ddb,37267,1731751077844, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=697cd6a363e514bff0d4ec954fc4034b, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:00:11,770 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731751192181.697cd6a363e514bff0d4ec954fc4034b., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=697cd6a363e514bff0d4ec954fc4034b, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:00:11,770 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731751192181.697cd6a363e514bff0d4ec954fc4034b., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 because the exception is null or not the one we care about 2024-11-16T10:00:12,009 INFO [RS_FLUSH_OPERATIONS-regionserver/037c58a19ddb:0-0 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=13.42 KB at sequenceid=365 (bloomFilter=true), to=hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/697cd6a363e514bff0d4ec954fc4034b/.tmp/B/743c3e50c2dc453da9f17314fa21a38a 2024-11-16T10:00:12,015 DEBUG [RS_FLUSH_OPERATIONS-regionserver/037c58a19ddb:0-0 {event_type=RS_FLUSH_REGIONS, pid=134}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/697cd6a363e514bff0d4ec954fc4034b/.tmp/C/e4d5f1ee2bca43e5adfbd16e2b3aa12a is 50, key is test_row_1/C:col10/1731751210222/Put/seqid=0 2024-11-16T10:00:12,018 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37591 is added to blk_1073742348_1524 (size=9857) 2024-11-16T10:00:12,077 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=697cd6a363e514bff0d4ec954fc4034b, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T10:00:12,077 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] ipc.CallRunner(138): callId: 151 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36058 deadline: 1731751272073, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=697cd6a363e514bff0d4ec954fc4034b, server=037c58a19ddb,37267,1731751077844 2024-11-16T10:00:12,078 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731751192181.697cd6a363e514bff0d4ec954fc4034b., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 , the old value is region=TestAcidGuarantees,,1731751192181.697cd6a363e514bff0d4ec954fc4034b., hostname=037c58a19ddb,37267,1731751077844, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=697cd6a363e514bff0d4ec954fc4034b, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:00:12,078 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731751192181.697cd6a363e514bff0d4ec954fc4034b., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=697cd6a363e514bff0d4ec954fc4034b, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:00:12,078 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731751192181.697cd6a363e514bff0d4ec954fc4034b., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 because the exception is null or not the one we care about 2024-11-16T10:00:12,085 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=697cd6a363e514bff0d4ec954fc4034b, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T10:00:12,086 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] ipc.CallRunner(138): callId: 138 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36022 deadline: 1731751272082, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=697cd6a363e514bff0d4ec954fc4034b, server=037c58a19ddb,37267,1731751077844 2024-11-16T10:00:12,086 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=697cd6a363e514bff0d4ec954fc4034b, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T10:00:12,086 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731751192181.697cd6a363e514bff0d4ec954fc4034b., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 , the old value is region=TestAcidGuarantees,,1731751192181.697cd6a363e514bff0d4ec954fc4034b., hostname=037c58a19ddb,37267,1731751077844, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=697cd6a363e514bff0d4ec954fc4034b, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:00:12,086 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] ipc.CallRunner(138): callId: 88 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36086 deadline: 1731751272082, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=697cd6a363e514bff0d4ec954fc4034b, server=037c58a19ddb,37267,1731751077844 2024-11-16T10:00:12,086 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731751192181.697cd6a363e514bff0d4ec954fc4034b., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=697cd6a363e514bff0d4ec954fc4034b, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:00:12,086 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731751192181.697cd6a363e514bff0d4ec954fc4034b., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 because the exception is null or not the one we care about 2024-11-16T10:00:12,086 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=697cd6a363e514bff0d4ec954fc4034b, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T10:00:12,086 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731751192181.697cd6a363e514bff0d4ec954fc4034b., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 , the old value is region=TestAcidGuarantees,,1731751192181.697cd6a363e514bff0d4ec954fc4034b., hostname=037c58a19ddb,37267,1731751077844, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=697cd6a363e514bff0d4ec954fc4034b, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:00:12,086 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731751192181.697cd6a363e514bff0d4ec954fc4034b., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=697cd6a363e514bff0d4ec954fc4034b, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:00:12,086 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] ipc.CallRunner(138): callId: 144 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36012 deadline: 1731751272083, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=697cd6a363e514bff0d4ec954fc4034b, server=037c58a19ddb,37267,1731751077844 2024-11-16T10:00:12,086 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731751192181.697cd6a363e514bff0d4ec954fc4034b., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 because the exception is null or not the one we care about 2024-11-16T10:00:12,086 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731751192181.697cd6a363e514bff0d4ec954fc4034b., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 , the old value is region=TestAcidGuarantees,,1731751192181.697cd6a363e514bff0d4ec954fc4034b., hostname=037c58a19ddb,37267,1731751077844, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=697cd6a363e514bff0d4ec954fc4034b, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:00:12,086 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731751192181.697cd6a363e514bff0d4ec954fc4034b., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=697cd6a363e514bff0d4ec954fc4034b, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:00:12,086 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731751192181.697cd6a363e514bff0d4ec954fc4034b., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 because the exception is null or not the one we care about 2024-11-16T10:00:12,419 INFO [RS_FLUSH_OPERATIONS-regionserver/037c58a19ddb:0-0 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=13.42 KB at sequenceid=365 (bloomFilter=true), to=hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/697cd6a363e514bff0d4ec954fc4034b/.tmp/C/e4d5f1ee2bca43e5adfbd16e2b3aa12a 2024-11-16T10:00:12,423 DEBUG [RS_FLUSH_OPERATIONS-regionserver/037c58a19ddb:0-0 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/697cd6a363e514bff0d4ec954fc4034b/.tmp/A/cdb34818a22c4599969803da6533b889 as hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/697cd6a363e514bff0d4ec954fc4034b/A/cdb34818a22c4599969803da6533b889 2024-11-16T10:00:12,426 INFO [RS_FLUSH_OPERATIONS-regionserver/037c58a19ddb:0-0 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/697cd6a363e514bff0d4ec954fc4034b/A/cdb34818a22c4599969803da6533b889, entries=100, sequenceid=365, filesize=9.6 K 2024-11-16T10:00:12,427 DEBUG [RS_FLUSH_OPERATIONS-regionserver/037c58a19ddb:0-0 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/697cd6a363e514bff0d4ec954fc4034b/.tmp/B/743c3e50c2dc453da9f17314fa21a38a as hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/697cd6a363e514bff0d4ec954fc4034b/B/743c3e50c2dc453da9f17314fa21a38a 2024-11-16T10:00:12,430 INFO [RS_FLUSH_OPERATIONS-regionserver/037c58a19ddb:0-0 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/697cd6a363e514bff0d4ec954fc4034b/B/743c3e50c2dc453da9f17314fa21a38a, entries=100, sequenceid=365, filesize=9.6 K 2024-11-16T10:00:12,430 DEBUG [RS_FLUSH_OPERATIONS-regionserver/037c58a19ddb:0-0 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/697cd6a363e514bff0d4ec954fc4034b/.tmp/C/e4d5f1ee2bca43e5adfbd16e2b3aa12a as hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/697cd6a363e514bff0d4ec954fc4034b/C/e4d5f1ee2bca43e5adfbd16e2b3aa12a 2024-11-16T10:00:12,434 INFO [RS_FLUSH_OPERATIONS-regionserver/037c58a19ddb:0-0 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/697cd6a363e514bff0d4ec954fc4034b/C/e4d5f1ee2bca43e5adfbd16e2b3aa12a, entries=100, sequenceid=365, filesize=9.6 K 2024-11-16T10:00:12,435 INFO [RS_FLUSH_OPERATIONS-regionserver/037c58a19ddb:0-0 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.HRegion(3140): Finished flush of dataSize ~40.25 KB/41220, heapSize ~106.17 KB/108720, currentSize=161.02 KB/164880 for 697cd6a363e514bff0d4ec954fc4034b in 1244ms, sequenceid=365, compaction requested=false 2024-11-16T10:00:12,435 DEBUG [RS_FLUSH_OPERATIONS-regionserver/037c58a19ddb:0-0 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.HRegion(2603): Flush status journal for 697cd6a363e514bff0d4ec954fc4034b: 2024-11-16T10:00:12,435 DEBUG [RS_FLUSH_OPERATIONS-regionserver/037c58a19ddb:0-0 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731751192181.697cd6a363e514bff0d4ec954fc4034b. 2024-11-16T10:00:12,435 DEBUG [RS_FLUSH_OPERATIONS-regionserver/037c58a19ddb:0-0 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=134 2024-11-16T10:00:12,435 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37877 {}] master.HMaster(4169): Remote procedure done, pid=134 2024-11-16T10:00:12,438 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=134, resume processing ppid=133 2024-11-16T10:00:12,438 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=134, ppid=133, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.0100 sec 2024-11-16T10:00:12,440 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=133, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=133, table=TestAcidGuarantees in 2.0160 sec 2024-11-16T10:00:12,573 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37877 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=133 2024-11-16T10:00:12,573 INFO [RPCClient-NioEventLoopGroup-4-10 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-16T10:00:12,574 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37877 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-16T10:00:12,575 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37877 {}] procedure2.ProcedureExecutor(1139): Stored pid=135, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=135, table=TestAcidGuarantees 2024-11-16T10:00:12,576 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=135, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=135, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-16T10:00:12,576 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37877 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=135 2024-11-16T10:00:12,577 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=135, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=135, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-16T10:00:12,577 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=136, ppid=135, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-16T10:00:12,597 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] regionserver.HRegion(8855): Flush requested on 697cd6a363e514bff0d4ec954fc4034b 2024-11-16T10:00:12,597 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-16T10:00:12,597 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 697cd6a363e514bff0d4ec954fc4034b 3/3 column families, dataSize=167.72 KB heapSize=440.20 KB 2024-11-16T10:00:12,603 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/697cd6a363e514bff0d4ec954fc4034b/.tmp/A/ca8e96abf12a4adfb357b4e54e388818 is 50, key is test_row_0/A:col10/1731751211438/Put/seqid=0 2024-11-16T10:00:12,606 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37591 is added to blk_1073742349_1525 (size=17181) 2024-11-16T10:00:12,617 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=697cd6a363e514bff0d4ec954fc4034b, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T10:00:12,617 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] ipc.CallRunner(138): callId: 154 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36058 deadline: 1731751272608, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=697cd6a363e514bff0d4ec954fc4034b, server=037c58a19ddb,37267,1731751077844 2024-11-16T10:00:12,617 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731751192181.697cd6a363e514bff0d4ec954fc4034b., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 , the old value is region=TestAcidGuarantees,,1731751192181.697cd6a363e514bff0d4ec954fc4034b., hostname=037c58a19ddb,37267,1731751077844, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=697cd6a363e514bff0d4ec954fc4034b, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:00:12,617 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731751192181.697cd6a363e514bff0d4ec954fc4034b., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=697cd6a363e514bff0d4ec954fc4034b, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:00:12,618 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731751192181.697cd6a363e514bff0d4ec954fc4034b., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 because the exception is null or not the one we care about 2024-11-16T10:00:12,618 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=697cd6a363e514bff0d4ec954fc4034b, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T10:00:12,618 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] ipc.CallRunner(138): callId: 141 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36022 deadline: 1731751272610, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=697cd6a363e514bff0d4ec954fc4034b, server=037c58a19ddb,37267,1731751077844 2024-11-16T10:00:12,618 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=697cd6a363e514bff0d4ec954fc4034b, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T10:00:12,619 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731751192181.697cd6a363e514bff0d4ec954fc4034b., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 , the old value is region=TestAcidGuarantees,,1731751192181.697cd6a363e514bff0d4ec954fc4034b., hostname=037c58a19ddb,37267,1731751077844, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=697cd6a363e514bff0d4ec954fc4034b, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:00:12,619 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] ipc.CallRunner(138): callId: 91 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36086 deadline: 1731751272610, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=697cd6a363e514bff0d4ec954fc4034b, server=037c58a19ddb,37267,1731751077844 2024-11-16T10:00:12,619 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731751192181.697cd6a363e514bff0d4ec954fc4034b., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=697cd6a363e514bff0d4ec954fc4034b, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:00:12,619 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731751192181.697cd6a363e514bff0d4ec954fc4034b., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 because the exception is null or not the one we care about 2024-11-16T10:00:12,619 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=697cd6a363e514bff0d4ec954fc4034b, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T10:00:12,619 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731751192181.697cd6a363e514bff0d4ec954fc4034b., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 , the old value is region=TestAcidGuarantees,,1731751192181.697cd6a363e514bff0d4ec954fc4034b., hostname=037c58a19ddb,37267,1731751077844, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=697cd6a363e514bff0d4ec954fc4034b, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:00:12,619 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] ipc.CallRunner(138): callId: 147 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36012 deadline: 1731751272610, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=697cd6a363e514bff0d4ec954fc4034b, server=037c58a19ddb,37267,1731751077844 2024-11-16T10:00:12,619 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731751192181.697cd6a363e514bff0d4ec954fc4034b., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=697cd6a363e514bff0d4ec954fc4034b, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:00:12,619 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731751192181.697cd6a363e514bff0d4ec954fc4034b., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 because the exception is null or not the one we care about 2024-11-16T10:00:12,619 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731751192181.697cd6a363e514bff0d4ec954fc4034b., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 , the old value is region=TestAcidGuarantees,,1731751192181.697cd6a363e514bff0d4ec954fc4034b., hostname=037c58a19ddb,37267,1731751077844, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=697cd6a363e514bff0d4ec954fc4034b, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:00:12,619 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731751192181.697cd6a363e514bff0d4ec954fc4034b., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=697cd6a363e514bff0d4ec954fc4034b, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:00:12,619 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731751192181.697cd6a363e514bff0d4ec954fc4034b., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 because the exception is null or not the one we care about 2024-11-16T10:00:12,683 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37877 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=135 2024-11-16T10:00:12,729 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=37267 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=136 2024-11-16T10:00:12,729 DEBUG [RS_FLUSH_OPERATIONS-regionserver/037c58a19ddb:0-1 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731751192181.697cd6a363e514bff0d4ec954fc4034b. 2024-11-16T10:00:12,729 DEBUG [RS_FLUSH_OPERATIONS-regionserver/037c58a19ddb:0-1 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731751192181.697cd6a363e514bff0d4ec954fc4034b. as already flushing 2024-11-16T10:00:12,729 DEBUG [RS_FLUSH_OPERATIONS-regionserver/037c58a19ddb:0-1 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731751192181.697cd6a363e514bff0d4ec954fc4034b. 2024-11-16T10:00:12,729 ERROR [RS_FLUSH_OPERATIONS-regionserver/037c58a19ddb:0-1 {event_type=RS_FLUSH_REGIONS, pid=136}] handler.RSProcedureHandler(58): pid=136 java.io.IOException: Unable to complete flush {ENCODED => 697cd6a363e514bff0d4ec954fc4034b, NAME => 'TestAcidGuarantees,,1731751192181.697cd6a363e514bff0d4ec954fc4034b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T10:00:12,730 DEBUG [RS_FLUSH_OPERATIONS-regionserver/037c58a19ddb:0-1 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=136 java.io.IOException: Unable to complete flush {ENCODED => 697cd6a363e514bff0d4ec954fc4034b, NAME => 'TestAcidGuarantees,,1731751192181.697cd6a363e514bff0d4ec954fc4034b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T10:00:12,730 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37877 {}] master.HMaster(4177): Remote procedure failed, pid=136 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 697cd6a363e514bff0d4ec954fc4034b, NAME => 'TestAcidGuarantees,,1731751192181.697cd6a363e514bff0d4ec954fc4034b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 697cd6a363e514bff0d4ec954fc4034b, NAME => 'TestAcidGuarantees,,1731751192181.697cd6a363e514bff0d4ec954fc4034b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T10:00:12,730 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=697cd6a363e514bff0d4ec954fc4034b, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T10:00:12,730 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] ipc.CallRunner(138): callId: 155 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36058 deadline: 1731751272723, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=697cd6a363e514bff0d4ec954fc4034b, server=037c58a19ddb,37267,1731751077844 2024-11-16T10:00:12,731 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=697cd6a363e514bff0d4ec954fc4034b, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T10:00:12,731 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731751192181.697cd6a363e514bff0d4ec954fc4034b., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 , the old value is region=TestAcidGuarantees,,1731751192181.697cd6a363e514bff0d4ec954fc4034b., hostname=037c58a19ddb,37267,1731751077844, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=697cd6a363e514bff0d4ec954fc4034b, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:00:12,731 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731751192181.697cd6a363e514bff0d4ec954fc4034b., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=697cd6a363e514bff0d4ec954fc4034b, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:00:12,731 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] ipc.CallRunner(138): callId: 142 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36022 deadline: 1731751272723, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=697cd6a363e514bff0d4ec954fc4034b, server=037c58a19ddb,37267,1731751077844 2024-11-16T10:00:12,731 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731751192181.697cd6a363e514bff0d4ec954fc4034b., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 because the exception is null or not the one we care about 2024-11-16T10:00:12,731 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731751192181.697cd6a363e514bff0d4ec954fc4034b., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 , the old value is region=TestAcidGuarantees,,1731751192181.697cd6a363e514bff0d4ec954fc4034b., hostname=037c58a19ddb,37267,1731751077844, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=697cd6a363e514bff0d4ec954fc4034b, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:00:12,731 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=697cd6a363e514bff0d4ec954fc4034b, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T10:00:12,731 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731751192181.697cd6a363e514bff0d4ec954fc4034b., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=697cd6a363e514bff0d4ec954fc4034b, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:00:12,731 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731751192181.697cd6a363e514bff0d4ec954fc4034b., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 because the exception is null or not the one we care about 2024-11-16T10:00:12,731 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] ipc.CallRunner(138): callId: 92 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36086 deadline: 1731751272723, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=697cd6a363e514bff0d4ec954fc4034b, server=037c58a19ddb,37267,1731751077844 2024-11-16T10:00:12,731 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731751192181.697cd6a363e514bff0d4ec954fc4034b., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 , the old value is region=TestAcidGuarantees,,1731751192181.697cd6a363e514bff0d4ec954fc4034b., hostname=037c58a19ddb,37267,1731751077844, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=697cd6a363e514bff0d4ec954fc4034b, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:00:12,731 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=697cd6a363e514bff0d4ec954fc4034b, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T10:00:12,731 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731751192181.697cd6a363e514bff0d4ec954fc4034b., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=697cd6a363e514bff0d4ec954fc4034b, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:00:12,731 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731751192181.697cd6a363e514bff0d4ec954fc4034b., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 because the exception is null or not the one we care about 2024-11-16T10:00:12,731 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] ipc.CallRunner(138): callId: 148 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36012 deadline: 1731751272723, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=697cd6a363e514bff0d4ec954fc4034b, server=037c58a19ddb,37267,1731751077844 2024-11-16T10:00:12,732 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731751192181.697cd6a363e514bff0d4ec954fc4034b., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 , the old value is region=TestAcidGuarantees,,1731751192181.697cd6a363e514bff0d4ec954fc4034b., hostname=037c58a19ddb,37267,1731751077844, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=697cd6a363e514bff0d4ec954fc4034b, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:00:12,732 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731751192181.697cd6a363e514bff0d4ec954fc4034b., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=697cd6a363e514bff0d4ec954fc4034b, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:00:12,732 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731751192181.697cd6a363e514bff0d4ec954fc4034b., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 because the exception is null or not the one we care about 2024-11-16T10:00:12,882 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=37267 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=136 2024-11-16T10:00:12,882 DEBUG [RS_FLUSH_OPERATIONS-regionserver/037c58a19ddb:0-2 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731751192181.697cd6a363e514bff0d4ec954fc4034b. 2024-11-16T10:00:12,882 DEBUG [RS_FLUSH_OPERATIONS-regionserver/037c58a19ddb:0-2 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731751192181.697cd6a363e514bff0d4ec954fc4034b. as already flushing 2024-11-16T10:00:12,882 DEBUG [RS_FLUSH_OPERATIONS-regionserver/037c58a19ddb:0-2 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731751192181.697cd6a363e514bff0d4ec954fc4034b. 2024-11-16T10:00:12,882 ERROR [RS_FLUSH_OPERATIONS-regionserver/037c58a19ddb:0-2 {event_type=RS_FLUSH_REGIONS, pid=136}] handler.RSProcedureHandler(58): pid=136 java.io.IOException: Unable to complete flush {ENCODED => 697cd6a363e514bff0d4ec954fc4034b, NAME => 'TestAcidGuarantees,,1731751192181.697cd6a363e514bff0d4ec954fc4034b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T10:00:12,882 DEBUG [RS_FLUSH_OPERATIONS-regionserver/037c58a19ddb:0-2 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=136 java.io.IOException: Unable to complete flush {ENCODED => 697cd6a363e514bff0d4ec954fc4034b, NAME => 'TestAcidGuarantees,,1731751192181.697cd6a363e514bff0d4ec954fc4034b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T10:00:12,883 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37877 {}] master.HMaster(4177): Remote procedure failed, pid=136 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 697cd6a363e514bff0d4ec954fc4034b, NAME => 'TestAcidGuarantees,,1731751192181.697cd6a363e514bff0d4ec954fc4034b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 697cd6a363e514bff0d4ec954fc4034b, NAME => 'TestAcidGuarantees,,1731751192181.697cd6a363e514bff0d4ec954fc4034b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T10:00:12,893 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37877 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=135 2024-11-16T10:00:12,936 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=697cd6a363e514bff0d4ec954fc4034b, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T10:00:12,936 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] ipc.CallRunner(138): callId: 143 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36022 deadline: 1731751272933, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=697cd6a363e514bff0d4ec954fc4034b, server=037c58a19ddb,37267,1731751077844 2024-11-16T10:00:12,936 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731751192181.697cd6a363e514bff0d4ec954fc4034b., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 , the old value is region=TestAcidGuarantees,,1731751192181.697cd6a363e514bff0d4ec954fc4034b., hostname=037c58a19ddb,37267,1731751077844, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=697cd6a363e514bff0d4ec954fc4034b, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:00:12,936 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731751192181.697cd6a363e514bff0d4ec954fc4034b., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=697cd6a363e514bff0d4ec954fc4034b, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:00:12,936 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731751192181.697cd6a363e514bff0d4ec954fc4034b., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 because the exception is null or not the one we care about 2024-11-16T10:00:12,946 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=697cd6a363e514bff0d4ec954fc4034b, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T10:00:12,946 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] ipc.CallRunner(138): callId: 156 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36058 deadline: 1731751272942, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=697cd6a363e514bff0d4ec954fc4034b, server=037c58a19ddb,37267,1731751077844 2024-11-16T10:00:12,946 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=697cd6a363e514bff0d4ec954fc4034b, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T10:00:12,946 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731751192181.697cd6a363e514bff0d4ec954fc4034b., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 , the old value is region=TestAcidGuarantees,,1731751192181.697cd6a363e514bff0d4ec954fc4034b., hostname=037c58a19ddb,37267,1731751077844, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=697cd6a363e514bff0d4ec954fc4034b, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:00:12,946 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731751192181.697cd6a363e514bff0d4ec954fc4034b., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=697cd6a363e514bff0d4ec954fc4034b, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:00:12,946 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] ipc.CallRunner(138): callId: 93 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36086 deadline: 1731751272943, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=697cd6a363e514bff0d4ec954fc4034b, server=037c58a19ddb,37267,1731751077844 2024-11-16T10:00:12,946 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731751192181.697cd6a363e514bff0d4ec954fc4034b., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 because the exception is null or not the one we care about 2024-11-16T10:00:12,946 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731751192181.697cd6a363e514bff0d4ec954fc4034b., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 , the old value is region=TestAcidGuarantees,,1731751192181.697cd6a363e514bff0d4ec954fc4034b., hostname=037c58a19ddb,37267,1731751077844, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=697cd6a363e514bff0d4ec954fc4034b, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:00:12,946 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=697cd6a363e514bff0d4ec954fc4034b, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T10:00:12,946 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731751192181.697cd6a363e514bff0d4ec954fc4034b., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=697cd6a363e514bff0d4ec954fc4034b, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:00:12,946 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731751192181.697cd6a363e514bff0d4ec954fc4034b., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 because the exception is null or not the one we care about 2024-11-16T10:00:12,946 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] ipc.CallRunner(138): callId: 149 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36012 deadline: 1731751272943, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=697cd6a363e514bff0d4ec954fc4034b, server=037c58a19ddb,37267,1731751077844 2024-11-16T10:00:12,947 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731751192181.697cd6a363e514bff0d4ec954fc4034b., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 , the old value is region=TestAcidGuarantees,,1731751192181.697cd6a363e514bff0d4ec954fc4034b., hostname=037c58a19ddb,37267,1731751077844, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=697cd6a363e514bff0d4ec954fc4034b, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:00:12,947 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731751192181.697cd6a363e514bff0d4ec954fc4034b., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=697cd6a363e514bff0d4ec954fc4034b, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:00:12,947 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731751192181.697cd6a363e514bff0d4ec954fc4034b., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 because the exception is null or not the one we care about 2024-11-16T10:00:13,007 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=58.14 KB at sequenceid=395 (bloomFilter=true), to=hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/697cd6a363e514bff0d4ec954fc4034b/.tmp/A/ca8e96abf12a4adfb357b4e54e388818 2024-11-16T10:00:13,014 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/697cd6a363e514bff0d4ec954fc4034b/.tmp/B/00e2727159444e7b88ef9c3ed5a91da2 is 50, key is test_row_0/B:col10/1731751211438/Put/seqid=0 2024-11-16T10:00:13,018 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37591 is added to blk_1073742350_1526 (size=12301) 2024-11-16T10:00:13,018 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=58.14 KB at sequenceid=395 (bloomFilter=true), to=hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/697cd6a363e514bff0d4ec954fc4034b/.tmp/B/00e2727159444e7b88ef9c3ed5a91da2 2024-11-16T10:00:13,030 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/697cd6a363e514bff0d4ec954fc4034b/.tmp/C/2b55b0ed5f8942d1a3f6a2eb0a32fc72 is 50, key is test_row_0/C:col10/1731751211438/Put/seqid=0 2024-11-16T10:00:13,035 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=37267 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=136 2024-11-16T10:00:13,035 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37591 is added to blk_1073742351_1527 (size=12301) 2024-11-16T10:00:13,038 DEBUG [RS_FLUSH_OPERATIONS-regionserver/037c58a19ddb:0-0 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731751192181.697cd6a363e514bff0d4ec954fc4034b. 2024-11-16T10:00:13,038 DEBUG [RS_FLUSH_OPERATIONS-regionserver/037c58a19ddb:0-0 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731751192181.697cd6a363e514bff0d4ec954fc4034b. as already flushing 2024-11-16T10:00:13,038 DEBUG [RS_FLUSH_OPERATIONS-regionserver/037c58a19ddb:0-0 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731751192181.697cd6a363e514bff0d4ec954fc4034b. 2024-11-16T10:00:13,038 ERROR [RS_FLUSH_OPERATIONS-regionserver/037c58a19ddb:0-0 {event_type=RS_FLUSH_REGIONS, pid=136}] handler.RSProcedureHandler(58): pid=136 java.io.IOException: Unable to complete flush {ENCODED => 697cd6a363e514bff0d4ec954fc4034b, NAME => 'TestAcidGuarantees,,1731751192181.697cd6a363e514bff0d4ec954fc4034b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T10:00:13,038 DEBUG [RS_FLUSH_OPERATIONS-regionserver/037c58a19ddb:0-0 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=136 java.io.IOException: Unable to complete flush {ENCODED => 697cd6a363e514bff0d4ec954fc4034b, NAME => 'TestAcidGuarantees,,1731751192181.697cd6a363e514bff0d4ec954fc4034b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T10:00:13,039 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=58.14 KB at sequenceid=395 (bloomFilter=true), to=hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/697cd6a363e514bff0d4ec954fc4034b/.tmp/C/2b55b0ed5f8942d1a3f6a2eb0a32fc72 2024-11-16T10:00:13,042 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37877 {}] master.HMaster(4177): Remote procedure failed, pid=136 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 697cd6a363e514bff0d4ec954fc4034b, NAME => 'TestAcidGuarantees,,1731751192181.697cd6a363e514bff0d4ec954fc4034b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 697cd6a363e514bff0d4ec954fc4034b, NAME => 'TestAcidGuarantees,,1731751192181.697cd6a363e514bff0d4ec954fc4034b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T10:00:13,048 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/697cd6a363e514bff0d4ec954fc4034b/.tmp/A/ca8e96abf12a4adfb357b4e54e388818 as hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/697cd6a363e514bff0d4ec954fc4034b/A/ca8e96abf12a4adfb357b4e54e388818 2024-11-16T10:00:13,053 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/697cd6a363e514bff0d4ec954fc4034b/A/ca8e96abf12a4adfb357b4e54e388818, entries=250, sequenceid=395, filesize=16.8 K 2024-11-16T10:00:13,054 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/697cd6a363e514bff0d4ec954fc4034b/.tmp/B/00e2727159444e7b88ef9c3ed5a91da2 as hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/697cd6a363e514bff0d4ec954fc4034b/B/00e2727159444e7b88ef9c3ed5a91da2 2024-11-16T10:00:13,057 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/697cd6a363e514bff0d4ec954fc4034b/B/00e2727159444e7b88ef9c3ed5a91da2, entries=150, sequenceid=395, filesize=12.0 K 2024-11-16T10:00:13,059 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/697cd6a363e514bff0d4ec954fc4034b/.tmp/C/2b55b0ed5f8942d1a3f6a2eb0a32fc72 as hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/697cd6a363e514bff0d4ec954fc4034b/C/2b55b0ed5f8942d1a3f6a2eb0a32fc72 2024-11-16T10:00:13,063 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/697cd6a363e514bff0d4ec954fc4034b/C/2b55b0ed5f8942d1a3f6a2eb0a32fc72, entries=150, sequenceid=395, filesize=12.0 K 2024-11-16T10:00:13,064 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~174.43 KB/178620, heapSize ~457.73 KB/468720, currentSize=40.25 KB/41220 for 697cd6a363e514bff0d4ec954fc4034b in 467ms, sequenceid=395, compaction requested=true 2024-11-16T10:00:13,064 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 697cd6a363e514bff0d4ec954fc4034b: 2024-11-16T10:00:13,064 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 697cd6a363e514bff0d4ec954fc4034b:A, priority=-2147483648, current under compaction store size is 1 2024-11-16T10:00:13,064 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-16T10:00:13,064 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 697cd6a363e514bff0d4ec954fc4034b:B, priority=-2147483648, current under compaction store size is 2 2024-11-16T10:00:13,064 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-16T10:00:13,064 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 697cd6a363e514bff0d4ec954fc4034b:C, priority=-2147483648, current under compaction store size is 3 2024-11-16T10:00:13,064 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-16T10:00:13,064 DEBUG [RS:0;037c58a19ddb:37267-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-16T10:00:13,064 DEBUG [RS:0;037c58a19ddb:37267-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-16T10:00:13,065 DEBUG [RS:0;037c58a19ddb:37267-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 35107 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-16T10:00:13,065 DEBUG [RS:0;037c58a19ddb:37267-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 39987 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-16T10:00:13,065 DEBUG [RS:0;037c58a19ddb:37267-longCompactions-0 {}] regionserver.HStore(1541): 697cd6a363e514bff0d4ec954fc4034b/B is initiating minor compaction (all files) 2024-11-16T10:00:13,065 DEBUG [RS:0;037c58a19ddb:37267-shortCompactions-0 {}] regionserver.HStore(1541): 697cd6a363e514bff0d4ec954fc4034b/A is initiating minor compaction (all files) 2024-11-16T10:00:13,065 INFO [RS:0;037c58a19ddb:37267-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 697cd6a363e514bff0d4ec954fc4034b/B in TestAcidGuarantees,,1731751192181.697cd6a363e514bff0d4ec954fc4034b. 2024-11-16T10:00:13,065 INFO [RS:0;037c58a19ddb:37267-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 697cd6a363e514bff0d4ec954fc4034b/A in TestAcidGuarantees,,1731751192181.697cd6a363e514bff0d4ec954fc4034b. 2024-11-16T10:00:13,065 INFO [RS:0;037c58a19ddb:37267-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/697cd6a363e514bff0d4ec954fc4034b/B/376083493aa8414fbc47f4cca9482837, hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/697cd6a363e514bff0d4ec954fc4034b/B/743c3e50c2dc453da9f17314fa21a38a, hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/697cd6a363e514bff0d4ec954fc4034b/B/00e2727159444e7b88ef9c3ed5a91da2] into tmpdir=hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/697cd6a363e514bff0d4ec954fc4034b/.tmp, totalSize=34.3 K 2024-11-16T10:00:13,065 INFO [RS:0;037c58a19ddb:37267-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/697cd6a363e514bff0d4ec954fc4034b/A/6287c88120694b81afbdd7d41cdc78ff, hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/697cd6a363e514bff0d4ec954fc4034b/A/cdb34818a22c4599969803da6533b889, hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/697cd6a363e514bff0d4ec954fc4034b/A/ca8e96abf12a4adfb357b4e54e388818] into tmpdir=hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/697cd6a363e514bff0d4ec954fc4034b/.tmp, totalSize=39.0 K 2024-11-16T10:00:13,066 DEBUG [RS:0;037c58a19ddb:37267-shortCompactions-0 {}] compactions.Compactor(225): Compacting 6287c88120694b81afbdd7d41cdc78ff, keycount=150, bloomtype=ROW, size=12.6 K, encoding=NONE, compression=NONE, seqNum=354, earliestPutTs=1731751209560 2024-11-16T10:00:13,066 DEBUG [RS:0;037c58a19ddb:37267-longCompactions-0 {}] compactions.Compactor(225): Compacting 376083493aa8414fbc47f4cca9482837, keycount=150, bloomtype=ROW, size=12.6 K, encoding=NONE, compression=NONE, seqNum=354, earliestPutTs=1731751209560 2024-11-16T10:00:13,068 DEBUG [RS:0;037c58a19ddb:37267-shortCompactions-0 {}] compactions.Compactor(225): Compacting cdb34818a22c4599969803da6533b889, keycount=100, bloomtype=ROW, size=9.6 K, encoding=NONE, compression=NONE, seqNum=365, earliestPutTs=1731751210218 2024-11-16T10:00:13,068 DEBUG [RS:0;037c58a19ddb:37267-shortCompactions-0 {}] compactions.Compactor(225): Compacting ca8e96abf12a4adfb357b4e54e388818, keycount=250, bloomtype=ROW, size=16.8 K, encoding=NONE, compression=NONE, seqNum=395, earliestPutTs=1731751211437 2024-11-16T10:00:13,069 DEBUG [RS:0;037c58a19ddb:37267-longCompactions-0 {}] compactions.Compactor(225): Compacting 743c3e50c2dc453da9f17314fa21a38a, keycount=100, bloomtype=ROW, size=9.6 K, encoding=NONE, compression=NONE, seqNum=365, earliestPutTs=1731751210218 2024-11-16T10:00:13,069 DEBUG [RS:0;037c58a19ddb:37267-longCompactions-0 {}] compactions.Compactor(225): Compacting 00e2727159444e7b88ef9c3ed5a91da2, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=395, earliestPutTs=1731751211438 2024-11-16T10:00:13,074 INFO [RS:0;037c58a19ddb:37267-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 697cd6a363e514bff0d4ec954fc4034b#A#compaction#454 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-16T10:00:13,074 DEBUG [RS:0;037c58a19ddb:37267-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/697cd6a363e514bff0d4ec954fc4034b/.tmp/A/3ffdd34e5ac64da0aad053a05f694142 is 50, key is test_row_0/A:col10/1731751211438/Put/seqid=0 2024-11-16T10:00:13,075 INFO [RS:0;037c58a19ddb:37267-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 697cd6a363e514bff0d4ec954fc4034b#B#compaction#455 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-16T10:00:13,075 DEBUG [RS:0;037c58a19ddb:37267-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/697cd6a363e514bff0d4ec954fc4034b/.tmp/B/7c6833f12db243bdb8260f018e1a1b8c is 50, key is test_row_0/B:col10/1731751211438/Put/seqid=0 2024-11-16T10:00:13,077 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37591 is added to blk_1073742352_1528 (size=13051) 2024-11-16T10:00:13,082 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37591 is added to blk_1073742353_1529 (size=13051) 2024-11-16T10:00:13,195 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=37267 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=136 2024-11-16T10:00:13,195 DEBUG [RS_FLUSH_OPERATIONS-regionserver/037c58a19ddb:0-1 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731751192181.697cd6a363e514bff0d4ec954fc4034b. 2024-11-16T10:00:13,195 INFO [RS_FLUSH_OPERATIONS-regionserver/037c58a19ddb:0-1 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.HRegion(2902): Flushing 697cd6a363e514bff0d4ec954fc4034b 3/3 column families, dataSize=40.25 KB heapSize=106.22 KB 2024-11-16T10:00:13,200 DEBUG [RS_FLUSH_OPERATIONS-regionserver/037c58a19ddb:0-1 {event_type=RS_FLUSH_REGIONS, pid=136}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/697cd6a363e514bff0d4ec954fc4034b/.tmp/A/9c2b874e4fd24e10ab7b089c59b0aeb0 is 50, key is test_row_0/A:col10/1731751212607/Put/seqid=0 2024-11-16T10:00:13,203 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37877 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=135 2024-11-16T10:00:13,203 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37591 is added to blk_1073742354_1530 (size=12301) 2024-11-16T10:00:13,251 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] regionserver.HRegion(8855): Flush requested on 697cd6a363e514bff0d4ec954fc4034b 2024-11-16T10:00:13,251 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731751192181.697cd6a363e514bff0d4ec954fc4034b. as already flushing 2024-11-16T10:00:13,312 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=697cd6a363e514bff0d4ec954fc4034b, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T10:00:13,313 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] ipc.CallRunner(138): callId: 99 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36086 deadline: 1731751273300, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=697cd6a363e514bff0d4ec954fc4034b, server=037c58a19ddb,37267,1731751077844 2024-11-16T10:00:13,313 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=697cd6a363e514bff0d4ec954fc4034b, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T10:00:13,313 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731751192181.697cd6a363e514bff0d4ec954fc4034b., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 , the old value is region=TestAcidGuarantees,,1731751192181.697cd6a363e514bff0d4ec954fc4034b., hostname=037c58a19ddb,37267,1731751077844, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=697cd6a363e514bff0d4ec954fc4034b, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:00:13,313 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731751192181.697cd6a363e514bff0d4ec954fc4034b., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=697cd6a363e514bff0d4ec954fc4034b, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:00:13,313 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] ipc.CallRunner(138): callId: 155 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36012 deadline: 1731751273309, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=697cd6a363e514bff0d4ec954fc4034b, server=037c58a19ddb,37267,1731751077844 2024-11-16T10:00:13,313 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731751192181.697cd6a363e514bff0d4ec954fc4034b., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 because the exception is null or not the one we care about 2024-11-16T10:00:13,313 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731751192181.697cd6a363e514bff0d4ec954fc4034b., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 , the old value is region=TestAcidGuarantees,,1731751192181.697cd6a363e514bff0d4ec954fc4034b., hostname=037c58a19ddb,37267,1731751077844, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=697cd6a363e514bff0d4ec954fc4034b, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:00:13,313 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731751192181.697cd6a363e514bff0d4ec954fc4034b., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=697cd6a363e514bff0d4ec954fc4034b, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:00:13,313 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731751192181.697cd6a363e514bff0d4ec954fc4034b., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 because the exception is null or not the one we care about 2024-11-16T10:00:13,320 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=697cd6a363e514bff0d4ec954fc4034b, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T10:00:13,320 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] ipc.CallRunner(138): callId: 152 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36022 deadline: 1731751273310, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=697cd6a363e514bff0d4ec954fc4034b, server=037c58a19ddb,37267,1731751077844 2024-11-16T10:00:13,320 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=697cd6a363e514bff0d4ec954fc4034b, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T10:00:13,320 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731751192181.697cd6a363e514bff0d4ec954fc4034b., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 , the old value is region=TestAcidGuarantees,,1731751192181.697cd6a363e514bff0d4ec954fc4034b., hostname=037c58a19ddb,37267,1731751077844, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=697cd6a363e514bff0d4ec954fc4034b, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:00:13,320 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731751192181.697cd6a363e514bff0d4ec954fc4034b., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=697cd6a363e514bff0d4ec954fc4034b, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:00:13,320 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] ipc.CallRunner(138): callId: 163 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36058 deadline: 1731751273311, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=697cd6a363e514bff0d4ec954fc4034b, server=037c58a19ddb,37267,1731751077844 2024-11-16T10:00:13,320 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731751192181.697cd6a363e514bff0d4ec954fc4034b., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 because the exception is null or not the one we care about 2024-11-16T10:00:13,321 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731751192181.697cd6a363e514bff0d4ec954fc4034b., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 , the old value is region=TestAcidGuarantees,,1731751192181.697cd6a363e514bff0d4ec954fc4034b., hostname=037c58a19ddb,37267,1731751077844, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=697cd6a363e514bff0d4ec954fc4034b, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:00:13,321 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731751192181.697cd6a363e514bff0d4ec954fc4034b., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=697cd6a363e514bff0d4ec954fc4034b, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:00:13,321 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731751192181.697cd6a363e514bff0d4ec954fc4034b., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 because the exception is null or not the one we care about 2024-11-16T10:00:13,430 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=697cd6a363e514bff0d4ec954fc4034b, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T10:00:13,431 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] ipc.CallRunner(138): callId: 100 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36086 deadline: 1731751273423, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=697cd6a363e514bff0d4ec954fc4034b, server=037c58a19ddb,37267,1731751077844 2024-11-16T10:00:13,431 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=697cd6a363e514bff0d4ec954fc4034b, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T10:00:13,431 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731751192181.697cd6a363e514bff0d4ec954fc4034b., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 , the old value is region=TestAcidGuarantees,,1731751192181.697cd6a363e514bff0d4ec954fc4034b., hostname=037c58a19ddb,37267,1731751077844, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=697cd6a363e514bff0d4ec954fc4034b, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:00:13,431 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] ipc.CallRunner(138): callId: 156 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36012 deadline: 1731751273423, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=697cd6a363e514bff0d4ec954fc4034b, server=037c58a19ddb,37267,1731751077844 2024-11-16T10:00:13,431 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731751192181.697cd6a363e514bff0d4ec954fc4034b., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=697cd6a363e514bff0d4ec954fc4034b, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:00:13,431 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731751192181.697cd6a363e514bff0d4ec954fc4034b., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 because the exception is null or not the one we care about 2024-11-16T10:00:13,431 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=697cd6a363e514bff0d4ec954fc4034b, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T10:00:13,431 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731751192181.697cd6a363e514bff0d4ec954fc4034b., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 , the old value is region=TestAcidGuarantees,,1731751192181.697cd6a363e514bff0d4ec954fc4034b., hostname=037c58a19ddb,37267,1731751077844, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=697cd6a363e514bff0d4ec954fc4034b, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:00:13,431 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] ipc.CallRunner(138): callId: 153 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36022 deadline: 1731751273423, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=697cd6a363e514bff0d4ec954fc4034b, server=037c58a19ddb,37267,1731751077844 2024-11-16T10:00:13,431 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731751192181.697cd6a363e514bff0d4ec954fc4034b., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=697cd6a363e514bff0d4ec954fc4034b, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:00:13,431 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731751192181.697cd6a363e514bff0d4ec954fc4034b., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 because the exception is null or not the one we care about 2024-11-16T10:00:13,431 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=697cd6a363e514bff0d4ec954fc4034b, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T10:00:13,431 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731751192181.697cd6a363e514bff0d4ec954fc4034b., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 , the old value is region=TestAcidGuarantees,,1731751192181.697cd6a363e514bff0d4ec954fc4034b., hostname=037c58a19ddb,37267,1731751077844, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=697cd6a363e514bff0d4ec954fc4034b, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:00:13,431 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] ipc.CallRunner(138): callId: 164 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36058 deadline: 1731751273423, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=697cd6a363e514bff0d4ec954fc4034b, server=037c58a19ddb,37267,1731751077844 2024-11-16T10:00:13,431 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731751192181.697cd6a363e514bff0d4ec954fc4034b., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=697cd6a363e514bff0d4ec954fc4034b, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:00:13,431 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731751192181.697cd6a363e514bff0d4ec954fc4034b., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 because the exception is null or not the one we care about 2024-11-16T10:00:13,432 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731751192181.697cd6a363e514bff0d4ec954fc4034b., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 , the old value is region=TestAcidGuarantees,,1731751192181.697cd6a363e514bff0d4ec954fc4034b., hostname=037c58a19ddb,37267,1731751077844, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=697cd6a363e514bff0d4ec954fc4034b, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:00:13,432 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731751192181.697cd6a363e514bff0d4ec954fc4034b., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=697cd6a363e514bff0d4ec954fc4034b, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:00:13,432 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731751192181.697cd6a363e514bff0d4ec954fc4034b., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 because the exception is null or not the one we care about 2024-11-16T10:00:13,481 DEBUG [RS:0;037c58a19ddb:37267-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/697cd6a363e514bff0d4ec954fc4034b/.tmp/A/3ffdd34e5ac64da0aad053a05f694142 as hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/697cd6a363e514bff0d4ec954fc4034b/A/3ffdd34e5ac64da0aad053a05f694142 2024-11-16T10:00:13,485 INFO [RS:0;037c58a19ddb:37267-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 697cd6a363e514bff0d4ec954fc4034b/A of 697cd6a363e514bff0d4ec954fc4034b into 3ffdd34e5ac64da0aad053a05f694142(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-16T10:00:13,486 DEBUG [RS:0;037c58a19ddb:37267-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 697cd6a363e514bff0d4ec954fc4034b: 2024-11-16T10:00:13,486 INFO [RS:0;037c58a19ddb:37267-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731751192181.697cd6a363e514bff0d4ec954fc4034b., storeName=697cd6a363e514bff0d4ec954fc4034b/A, priority=13, startTime=1731751213064; duration=0sec 2024-11-16T10:00:13,486 DEBUG [RS:0;037c58a19ddb:37267-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-16T10:00:13,486 DEBUG [RS:0;037c58a19ddb:37267-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 697cd6a363e514bff0d4ec954fc4034b:A 2024-11-16T10:00:13,486 DEBUG [RS:0;037c58a19ddb:37267-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-16T10:00:13,486 DEBUG [RS:0;037c58a19ddb:37267-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/697cd6a363e514bff0d4ec954fc4034b/.tmp/B/7c6833f12db243bdb8260f018e1a1b8c as hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/697cd6a363e514bff0d4ec954fc4034b/B/7c6833f12db243bdb8260f018e1a1b8c 2024-11-16T10:00:13,487 DEBUG [RS:0;037c58a19ddb:37267-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 35107 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-16T10:00:13,487 DEBUG [RS:0;037c58a19ddb:37267-shortCompactions-0 {}] regionserver.HStore(1541): 697cd6a363e514bff0d4ec954fc4034b/C is initiating minor compaction (all files) 2024-11-16T10:00:13,487 INFO [RS:0;037c58a19ddb:37267-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 697cd6a363e514bff0d4ec954fc4034b/C in TestAcidGuarantees,,1731751192181.697cd6a363e514bff0d4ec954fc4034b. 2024-11-16T10:00:13,487 INFO [RS:0;037c58a19ddb:37267-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/697cd6a363e514bff0d4ec954fc4034b/C/0c90ee06996748789e1d25093a7f1e25, hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/697cd6a363e514bff0d4ec954fc4034b/C/e4d5f1ee2bca43e5adfbd16e2b3aa12a, hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/697cd6a363e514bff0d4ec954fc4034b/C/2b55b0ed5f8942d1a3f6a2eb0a32fc72] into tmpdir=hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/697cd6a363e514bff0d4ec954fc4034b/.tmp, totalSize=34.3 K 2024-11-16T10:00:13,487 DEBUG [RS:0;037c58a19ddb:37267-shortCompactions-0 {}] compactions.Compactor(225): Compacting 0c90ee06996748789e1d25093a7f1e25, keycount=150, bloomtype=ROW, size=12.6 K, encoding=NONE, compression=NONE, seqNum=354, earliestPutTs=1731751209560 2024-11-16T10:00:13,487 DEBUG [RS:0;037c58a19ddb:37267-shortCompactions-0 {}] compactions.Compactor(225): Compacting e4d5f1ee2bca43e5adfbd16e2b3aa12a, keycount=100, bloomtype=ROW, size=9.6 K, encoding=NONE, compression=NONE, seqNum=365, earliestPutTs=1731751210218 2024-11-16T10:00:13,488 DEBUG [RS:0;037c58a19ddb:37267-shortCompactions-0 {}] compactions.Compactor(225): Compacting 2b55b0ed5f8942d1a3f6a2eb0a32fc72, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=395, earliestPutTs=1731751211438 2024-11-16T10:00:13,490 INFO [RS:0;037c58a19ddb:37267-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 697cd6a363e514bff0d4ec954fc4034b/B of 697cd6a363e514bff0d4ec954fc4034b into 7c6833f12db243bdb8260f018e1a1b8c(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-16T10:00:13,490 DEBUG [RS:0;037c58a19ddb:37267-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 697cd6a363e514bff0d4ec954fc4034b: 2024-11-16T10:00:13,490 INFO [RS:0;037c58a19ddb:37267-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731751192181.697cd6a363e514bff0d4ec954fc4034b., storeName=697cd6a363e514bff0d4ec954fc4034b/B, priority=13, startTime=1731751213064; duration=0sec 2024-11-16T10:00:13,490 DEBUG [RS:0;037c58a19ddb:37267-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-16T10:00:13,490 DEBUG [RS:0;037c58a19ddb:37267-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 697cd6a363e514bff0d4ec954fc4034b:B 2024-11-16T10:00:13,494 INFO [RS:0;037c58a19ddb:37267-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 697cd6a363e514bff0d4ec954fc4034b#C#compaction#457 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-16T10:00:13,495 DEBUG [RS:0;037c58a19ddb:37267-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/697cd6a363e514bff0d4ec954fc4034b/.tmp/C/344c6db7c2164b52a9e08f25363cf264 is 50, key is test_row_0/C:col10/1731751211438/Put/seqid=0 2024-11-16T10:00:13,499 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37591 is added to blk_1073742355_1531 (size=13051) 2024-11-16T10:00:13,604 INFO [RS_FLUSH_OPERATIONS-regionserver/037c58a19ddb:0-1 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=13.42 KB at sequenceid=404 (bloomFilter=true), to=hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/697cd6a363e514bff0d4ec954fc4034b/.tmp/A/9c2b874e4fd24e10ab7b089c59b0aeb0 2024-11-16T10:00:13,610 DEBUG [RS_FLUSH_OPERATIONS-regionserver/037c58a19ddb:0-1 {event_type=RS_FLUSH_REGIONS, pid=136}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/697cd6a363e514bff0d4ec954fc4034b/.tmp/B/a7655708fdb94d7fb618714a81f63a33 is 50, key is test_row_0/B:col10/1731751212607/Put/seqid=0 2024-11-16T10:00:13,613 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37591 is added to blk_1073742356_1532 (size=12301) 2024-11-16T10:00:13,646 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=697cd6a363e514bff0d4ec954fc4034b, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T10:00:13,647 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] ipc.CallRunner(138): callId: 101 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36086 deadline: 1731751273643, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=697cd6a363e514bff0d4ec954fc4034b, server=037c58a19ddb,37267,1731751077844 2024-11-16T10:00:13,647 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731751192181.697cd6a363e514bff0d4ec954fc4034b., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 , the old value is region=TestAcidGuarantees,,1731751192181.697cd6a363e514bff0d4ec954fc4034b., hostname=037c58a19ddb,37267,1731751077844, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=697cd6a363e514bff0d4ec954fc4034b, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:00:13,647 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731751192181.697cd6a363e514bff0d4ec954fc4034b., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=697cd6a363e514bff0d4ec954fc4034b, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:00:13,647 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731751192181.697cd6a363e514bff0d4ec954fc4034b., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 because the exception is null or not the one we care about 2024-11-16T10:00:13,652 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=697cd6a363e514bff0d4ec954fc4034b, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T10:00:13,652 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] ipc.CallRunner(138): callId: 154 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36022 deadline: 1731751273644, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=697cd6a363e514bff0d4ec954fc4034b, server=037c58a19ddb,37267,1731751077844 2024-11-16T10:00:13,652 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=697cd6a363e514bff0d4ec954fc4034b, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T10:00:13,653 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731751192181.697cd6a363e514bff0d4ec954fc4034b., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 , the old value is region=TestAcidGuarantees,,1731751192181.697cd6a363e514bff0d4ec954fc4034b., hostname=037c58a19ddb,37267,1731751077844, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=697cd6a363e514bff0d4ec954fc4034b, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:00:13,653 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731751192181.697cd6a363e514bff0d4ec954fc4034b., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=697cd6a363e514bff0d4ec954fc4034b, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:00:13,653 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] ipc.CallRunner(138): callId: 157 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36012 deadline: 1731751273644, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=697cd6a363e514bff0d4ec954fc4034b, server=037c58a19ddb,37267,1731751077844 2024-11-16T10:00:13,653 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731751192181.697cd6a363e514bff0d4ec954fc4034b., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 because the exception is null or not the one we care about 2024-11-16T10:00:13,653 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731751192181.697cd6a363e514bff0d4ec954fc4034b., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 , the old value is region=TestAcidGuarantees,,1731751192181.697cd6a363e514bff0d4ec954fc4034b., hostname=037c58a19ddb,37267,1731751077844, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=697cd6a363e514bff0d4ec954fc4034b, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:00:13,653 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=697cd6a363e514bff0d4ec954fc4034b, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T10:00:13,653 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731751192181.697cd6a363e514bff0d4ec954fc4034b., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=697cd6a363e514bff0d4ec954fc4034b, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:00:13,653 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731751192181.697cd6a363e514bff0d4ec954fc4034b., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 because the exception is null or not the one we care about 2024-11-16T10:00:13,653 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] ipc.CallRunner(138): callId: 165 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36058 deadline: 1731751273644, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=697cd6a363e514bff0d4ec954fc4034b, server=037c58a19ddb,37267,1731751077844 2024-11-16T10:00:13,653 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731751192181.697cd6a363e514bff0d4ec954fc4034b., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 , the old value is region=TestAcidGuarantees,,1731751192181.697cd6a363e514bff0d4ec954fc4034b., hostname=037c58a19ddb,37267,1731751077844, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=697cd6a363e514bff0d4ec954fc4034b, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:00:13,653 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731751192181.697cd6a363e514bff0d4ec954fc4034b., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=697cd6a363e514bff0d4ec954fc4034b, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:00:13,654 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731751192181.697cd6a363e514bff0d4ec954fc4034b., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 because the exception is null or not the one we care about 2024-11-16T10:00:13,713 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37877 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=135 2024-11-16T10:00:13,904 DEBUG [RS:0;037c58a19ddb:37267-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/697cd6a363e514bff0d4ec954fc4034b/.tmp/C/344c6db7c2164b52a9e08f25363cf264 as hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/697cd6a363e514bff0d4ec954fc4034b/C/344c6db7c2164b52a9e08f25363cf264 2024-11-16T10:00:13,908 INFO [RS:0;037c58a19ddb:37267-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 697cd6a363e514bff0d4ec954fc4034b/C of 697cd6a363e514bff0d4ec954fc4034b into 344c6db7c2164b52a9e08f25363cf264(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-16T10:00:13,908 DEBUG [RS:0;037c58a19ddb:37267-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 697cd6a363e514bff0d4ec954fc4034b: 2024-11-16T10:00:13,908 INFO [RS:0;037c58a19ddb:37267-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731751192181.697cd6a363e514bff0d4ec954fc4034b., storeName=697cd6a363e514bff0d4ec954fc4034b/C, priority=13, startTime=1731751213064; duration=0sec 2024-11-16T10:00:13,908 DEBUG [RS:0;037c58a19ddb:37267-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-16T10:00:13,908 DEBUG [RS:0;037c58a19ddb:37267-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 697cd6a363e514bff0d4ec954fc4034b:C 2024-11-16T10:00:13,956 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=697cd6a363e514bff0d4ec954fc4034b, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T10:00:13,957 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] ipc.CallRunner(138): callId: 102 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36086 deadline: 1731751273953, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=697cd6a363e514bff0d4ec954fc4034b, server=037c58a19ddb,37267,1731751077844 2024-11-16T10:00:13,957 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731751192181.697cd6a363e514bff0d4ec954fc4034b., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 , the old value is region=TestAcidGuarantees,,1731751192181.697cd6a363e514bff0d4ec954fc4034b., hostname=037c58a19ddb,37267,1731751077844, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=697cd6a363e514bff0d4ec954fc4034b, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:00:13,957 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731751192181.697cd6a363e514bff0d4ec954fc4034b., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=697cd6a363e514bff0d4ec954fc4034b, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:00:13,957 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731751192181.697cd6a363e514bff0d4ec954fc4034b., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 because the exception is null or not the one we care about 2024-11-16T10:00:13,966 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=697cd6a363e514bff0d4ec954fc4034b, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T10:00:13,966 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] ipc.CallRunner(138): callId: 155 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36022 deadline: 1731751273963, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=697cd6a363e514bff0d4ec954fc4034b, server=037c58a19ddb,37267,1731751077844 2024-11-16T10:00:13,967 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=697cd6a363e514bff0d4ec954fc4034b, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T10:00:13,967 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] ipc.CallRunner(138): callId: 158 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36012 deadline: 1731751273963, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=697cd6a363e514bff0d4ec954fc4034b, server=037c58a19ddb,37267,1731751077844 2024-11-16T10:00:13,967 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731751192181.697cd6a363e514bff0d4ec954fc4034b., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 , the old value is region=TestAcidGuarantees,,1731751192181.697cd6a363e514bff0d4ec954fc4034b., hostname=037c58a19ddb,37267,1731751077844, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=697cd6a363e514bff0d4ec954fc4034b, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:00:13,967 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731751192181.697cd6a363e514bff0d4ec954fc4034b., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=697cd6a363e514bff0d4ec954fc4034b, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:00:13,967 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731751192181.697cd6a363e514bff0d4ec954fc4034b., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 because the exception is null or not the one we care about 2024-11-16T10:00:13,967 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=697cd6a363e514bff0d4ec954fc4034b, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T10:00:13,967 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731751192181.697cd6a363e514bff0d4ec954fc4034b., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 , the old value is region=TestAcidGuarantees,,1731751192181.697cd6a363e514bff0d4ec954fc4034b., hostname=037c58a19ddb,37267,1731751077844, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=697cd6a363e514bff0d4ec954fc4034b, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:00:13,967 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] ipc.CallRunner(138): callId: 166 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36058 deadline: 1731751273963, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=697cd6a363e514bff0d4ec954fc4034b, server=037c58a19ddb,37267,1731751077844 2024-11-16T10:00:13,967 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731751192181.697cd6a363e514bff0d4ec954fc4034b., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=697cd6a363e514bff0d4ec954fc4034b, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:00:13,967 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731751192181.697cd6a363e514bff0d4ec954fc4034b., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 because the exception is null or not the one we care about 2024-11-16T10:00:13,967 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731751192181.697cd6a363e514bff0d4ec954fc4034b., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 , the old value is region=TestAcidGuarantees,,1731751192181.697cd6a363e514bff0d4ec954fc4034b., hostname=037c58a19ddb,37267,1731751077844, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=697cd6a363e514bff0d4ec954fc4034b, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:00:13,967 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731751192181.697cd6a363e514bff0d4ec954fc4034b., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=697cd6a363e514bff0d4ec954fc4034b, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:00:13,967 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731751192181.697cd6a363e514bff0d4ec954fc4034b., hostname=037c58a19ddb,37267,1731751077844, seqNum=2 because the exception is null or not the one we care about 2024-11-16T10:00:14,014 INFO [RS_FLUSH_OPERATIONS-regionserver/037c58a19ddb:0-1 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=13.42 KB at sequenceid=404 (bloomFilter=true), to=hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/697cd6a363e514bff0d4ec954fc4034b/.tmp/B/a7655708fdb94d7fb618714a81f63a33 2024-11-16T10:00:14,020 DEBUG [RS_FLUSH_OPERATIONS-regionserver/037c58a19ddb:0-1 {event_type=RS_FLUSH_REGIONS, pid=136}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/697cd6a363e514bff0d4ec954fc4034b/.tmp/C/8b495ff68e964d87919267942138b50c is 50, key is test_row_0/C:col10/1731751212607/Put/seqid=0 2024-11-16T10:00:14,023 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37591 is added to blk_1073742357_1533 (size=12301) 2024-11-16T10:00:14,371 INFO [Thread-2008 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2008. 2024-11-16T10:00:14,371 DEBUG [Thread-2008 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-16T10:00:14,371 DEBUG [Thread-2008 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T10:00:14,371 DEBUG [Thread-2008 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T10:00:14,371 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-16T10:00:14,375 INFO [Thread-2000 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2000. 2024-11-16T10:00:14,375 DEBUG [Thread-2000 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-16T10:00:14,375 DEBUG [Thread-2000 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T10:00:14,375 DEBUG [Thread-2000 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T10:00:14,375 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-16T10:00:14,378 INFO [Thread-2002 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2002. 2024-11-16T10:00:14,378 DEBUG [Thread-2002 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-16T10:00:14,378 DEBUG [Thread-2002 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T10:00:14,378 DEBUG [Thread-2002 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T10:00:14,378 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-16T10:00:14,379 INFO [Thread-2006 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2006. 2024-11-16T10:00:14,379 DEBUG [Thread-2006 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-16T10:00:14,379 DEBUG [Thread-2006 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T10:00:14,379 DEBUG [Thread-2006 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T10:00:14,379 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-16T10:00:14,382 INFO [Thread-2004 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2004. 2024-11-16T10:00:14,382 DEBUG [Thread-2004 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-16T10:00:14,382 DEBUG [Thread-2004 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T10:00:14,382 DEBUG [Thread-2004 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T10:00:14,382 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-16T10:00:14,424 INFO [RS_FLUSH_OPERATIONS-regionserver/037c58a19ddb:0-1 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=13.42 KB at sequenceid=404 (bloomFilter=true), to=hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/697cd6a363e514bff0d4ec954fc4034b/.tmp/C/8b495ff68e964d87919267942138b50c 2024-11-16T10:00:14,427 DEBUG [RS_FLUSH_OPERATIONS-regionserver/037c58a19ddb:0-1 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/697cd6a363e514bff0d4ec954fc4034b/.tmp/A/9c2b874e4fd24e10ab7b089c59b0aeb0 as hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/697cd6a363e514bff0d4ec954fc4034b/A/9c2b874e4fd24e10ab7b089c59b0aeb0 2024-11-16T10:00:14,429 INFO [RS_FLUSH_OPERATIONS-regionserver/037c58a19ddb:0-1 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/697cd6a363e514bff0d4ec954fc4034b/A/9c2b874e4fd24e10ab7b089c59b0aeb0, entries=150, sequenceid=404, filesize=12.0 K 2024-11-16T10:00:14,430 DEBUG [RS_FLUSH_OPERATIONS-regionserver/037c58a19ddb:0-1 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/697cd6a363e514bff0d4ec954fc4034b/.tmp/B/a7655708fdb94d7fb618714a81f63a33 as hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/697cd6a363e514bff0d4ec954fc4034b/B/a7655708fdb94d7fb618714a81f63a33 2024-11-16T10:00:14,432 INFO [RS_FLUSH_OPERATIONS-regionserver/037c58a19ddb:0-1 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/697cd6a363e514bff0d4ec954fc4034b/B/a7655708fdb94d7fb618714a81f63a33, entries=150, sequenceid=404, filesize=12.0 K 2024-11-16T10:00:14,433 DEBUG [RS_FLUSH_OPERATIONS-regionserver/037c58a19ddb:0-1 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/697cd6a363e514bff0d4ec954fc4034b/.tmp/C/8b495ff68e964d87919267942138b50c as hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/697cd6a363e514bff0d4ec954fc4034b/C/8b495ff68e964d87919267942138b50c 2024-11-16T10:00:14,435 INFO [RS_FLUSH_OPERATIONS-regionserver/037c58a19ddb:0-1 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/697cd6a363e514bff0d4ec954fc4034b/C/8b495ff68e964d87919267942138b50c, entries=150, sequenceid=404, filesize=12.0 K 2024-11-16T10:00:14,436 INFO [RS_FLUSH_OPERATIONS-regionserver/037c58a19ddb:0-1 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.HRegion(3140): Finished flush of dataSize ~40.25 KB/41220, heapSize ~106.17 KB/108720, currentSize=161.02 KB/164880 for 697cd6a363e514bff0d4ec954fc4034b in 1241ms, sequenceid=404, compaction requested=false 2024-11-16T10:00:14,436 DEBUG [RS_FLUSH_OPERATIONS-regionserver/037c58a19ddb:0-1 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.HRegion(2603): Flush status journal for 697cd6a363e514bff0d4ec954fc4034b: 2024-11-16T10:00:14,436 DEBUG [RS_FLUSH_OPERATIONS-regionserver/037c58a19ddb:0-1 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731751192181.697cd6a363e514bff0d4ec954fc4034b. 2024-11-16T10:00:14,436 DEBUG [RS_FLUSH_OPERATIONS-regionserver/037c58a19ddb:0-1 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=136 2024-11-16T10:00:14,437 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37877 {}] master.HMaster(4169): Remote procedure done, pid=136 2024-11-16T10:00:14,439 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=136, resume processing ppid=135 2024-11-16T10:00:14,439 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=136, ppid=135, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.8600 sec 2024-11-16T10:00:14,440 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=135, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=135, table=TestAcidGuarantees in 1.8650 sec 2024-11-16T10:00:14,464 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] regionserver.HRegion(8855): Flush requested on 697cd6a363e514bff0d4ec954fc4034b 2024-11-16T10:00:14,464 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-16T10:00:14,464 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 697cd6a363e514bff0d4ec954fc4034b 3/3 column families, dataSize=167.72 KB heapSize=440.20 KB 2024-11-16T10:00:14,464 INFO [Thread-1993 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1993. 2024-11-16T10:00:14,464 DEBUG [Thread-1993 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-16T10:00:14,464 DEBUG [Thread-1993 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T10:00:14,464 DEBUG [Thread-1993 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T10:00:14,464 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-16T10:00:14,467 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/697cd6a363e514bff0d4ec954fc4034b/.tmp/A/9fe9bcf9a3c74206a24135894d172f1c is 50, key is test_row_0/A:col10/1731751213307/Put/seqid=0 2024-11-16T10:00:14,470 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37591 is added to blk_1073742358_1534 (size=12301) 2024-11-16T10:00:14,474 INFO [Thread-1991 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1991. 2024-11-16T10:00:14,474 INFO [Thread-1989 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1989. 2024-11-16T10:00:14,474 DEBUG [Thread-1991 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-16T10:00:14,474 INFO [Thread-1997 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1997. 2024-11-16T10:00:14,474 DEBUG [Thread-1989 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-16T10:00:14,474 DEBUG [Thread-1991 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T10:00:14,474 DEBUG [Thread-1997 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-16T10:00:14,474 DEBUG [Thread-1989 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T10:00:14,474 DEBUG [Thread-1997 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T10:00:14,475 DEBUG [Thread-1997 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T10:00:14,475 DEBUG [Thread-1991 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T10:00:14,475 DEBUG [Thread-1989 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T10:00:14,475 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-16T10:00:14,475 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-16T10:00:14,475 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-16T10:00:14,723 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37877 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=135 2024-11-16T10:00:14,723 INFO [RPCClient-NioEventLoopGroup-4-10 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-16T10:00:14,871 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=55.91 KB at sequenceid=435 (bloomFilter=true), to=hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/697cd6a363e514bff0d4ec954fc4034b/.tmp/A/9fe9bcf9a3c74206a24135894d172f1c 2024-11-16T10:00:14,876 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/697cd6a363e514bff0d4ec954fc4034b/.tmp/B/8262e61da0be43eaacaeb6971c33d6ad is 50, key is test_row_0/B:col10/1731751213307/Put/seqid=0 2024-11-16T10:00:14,879 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37591 is added to blk_1073742359_1535 (size=12301) 2024-11-16T10:00:15,045 INFO [Thread-1995 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1995. 2024-11-16T10:00:15,045 DEBUG [Thread-1995 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-16T10:00:15,045 DEBUG [Thread-1995 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T10:00:15,045 DEBUG [Thread-1995 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T10:00:15,045 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-16T10:00:15,045 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(392): Finished test. Writers: 2024-11-16T10:00:15,045 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 100 2024-11-16T10:00:15,045 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 88 2024-11-16T10:00:15,045 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 55 2024-11-16T10:00:15,045 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 11 2024-11-16T10:00:15,045 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 91 2024-11-16T10:00:15,045 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(396): Readers: 2024-11-16T10:00:15,045 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(400): Scanners: 2024-11-16T10:00:15,045 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1495 2024-11-16T10:00:15,045 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 4483 rows 2024-11-16T10:00:15,045 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1499 2024-11-16T10:00:15,045 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 4495 rows 2024-11-16T10:00:15,045 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1501 2024-11-16T10:00:15,045 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 4501 rows 2024-11-16T10:00:15,045 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1500 2024-11-16T10:00:15,045 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 4498 rows 2024-11-16T10:00:15,045 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1511 2024-11-16T10:00:15,045 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 4528 rows 2024-11-16T10:00:15,045 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-16T10:00:15,045 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool.doWork(AcidGuaranteesTestTool.java:131) at org.apache.hadoop.hbase.util.AbstractHBaseTool.run(AbstractHBaseTool.java:150) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.runTestAtomicity(AcidGuaranteesTestBase.java:102) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.runTestAtomicity(AcidGuaranteesTestBase.java:91) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.testScanAtomicity(AcidGuaranteesTestBase.java:112) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-16T10:00:15,045 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T10:00:15,046 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T10:00:15,046 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-16T10:00:15,046 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37877 {}] master.HMaster$13(2820): Client=jenkins//172.17.0.2 disable TestAcidGuarantees 2024-11-16T10:00:15,047 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37877 {}] procedure2.ProcedureExecutor(1139): Stored pid=137, state=RUNNABLE:DISABLE_TABLE_PREPARE, hasLock=false; DisableTableProcedure table=TestAcidGuarantees 2024-11-16T10:00:15,048 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37877 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=137 2024-11-16T10:00:15,049 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731751215049"}]},"ts":"1731751215049"} 2024-11-16T10:00:15,050 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLING in hbase:meta 2024-11-16T10:00:15,050 INFO [PEWorker-3 {}] procedure.DisableTableProcedure(284): Set TestAcidGuarantees to state=DISABLING 2024-11-16T10:00:15,050 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=138, ppid=137, state=RUNNABLE:CLOSE_TABLE_REGIONS_SCHEDULE, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees}] 2024-11-16T10:00:15,052 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=139, ppid=138, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=697cd6a363e514bff0d4ec954fc4034b, UNASSIGN}] 2024-11-16T10:00:15,052 INFO [PEWorker-2 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=139, ppid=138, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=697cd6a363e514bff0d4ec954fc4034b, UNASSIGN 2024-11-16T10:00:15,053 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=139 updating hbase:meta row=697cd6a363e514bff0d4ec954fc4034b, regionState=CLOSING, regionLocation=037c58a19ddb,37267,1731751077844 2024-11-16T10:00:15,054 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=139, ppid=138, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=697cd6a363e514bff0d4ec954fc4034b, UNASSIGN because future has completed 2024-11-16T10:00:15,054 DEBUG [PEWorker-1 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-11-16T10:00:15,054 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=140, ppid=139, state=RUNNABLE, hasLock=false; CloseRegionProcedure 697cd6a363e514bff0d4ec954fc4034b, server=037c58a19ddb,37267,1731751077844}] 2024-11-16T10:00:15,153 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37877 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=137 2024-11-16T10:00:15,206 INFO [RS_CLOSE_REGION-regionserver/037c58a19ddb:0-0 {event_type=M_RS_CLOSE_REGION, pid=140}] handler.UnassignRegionHandler(122): Close 697cd6a363e514bff0d4ec954fc4034b 2024-11-16T10:00:15,206 DEBUG [RS_CLOSE_REGION-regionserver/037c58a19ddb:0-0 {event_type=M_RS_CLOSE_REGION, pid=140}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-11-16T10:00:15,206 DEBUG [RS_CLOSE_REGION-regionserver/037c58a19ddb:0-0 {event_type=M_RS_CLOSE_REGION, pid=140}] regionserver.HRegion(1722): Closing 697cd6a363e514bff0d4ec954fc4034b, disabling compactions & flushes 2024-11-16T10:00:15,206 DEBUG [RS_CLOSE_REGION-regionserver/037c58a19ddb:0-0 {event_type=M_RS_CLOSE_REGION, pid=140}] regionserver.HRegion(1993): waiting for 0 compactions & cache flush to complete for region TestAcidGuarantees,,1731751192181.697cd6a363e514bff0d4ec954fc4034b. 2024-11-16T10:00:15,279 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=55.91 KB at sequenceid=435 (bloomFilter=true), to=hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/697cd6a363e514bff0d4ec954fc4034b/.tmp/B/8262e61da0be43eaacaeb6971c33d6ad 2024-11-16T10:00:15,285 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/697cd6a363e514bff0d4ec954fc4034b/.tmp/C/add9632fbd004800979d682fb30d05a0 is 50, key is test_row_0/C:col10/1731751213307/Put/seqid=0 2024-11-16T10:00:15,288 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37591 is added to blk_1073742360_1536 (size=12301) 2024-11-16T10:00:15,362 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37877 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=137 2024-11-16T10:00:15,672 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37877 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=137 2024-11-16T10:00:15,688 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=55.91 KB at sequenceid=435 (bloomFilter=true), to=hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/697cd6a363e514bff0d4ec954fc4034b/.tmp/C/add9632fbd004800979d682fb30d05a0 2024-11-16T10:00:15,692 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/697cd6a363e514bff0d4ec954fc4034b/.tmp/A/9fe9bcf9a3c74206a24135894d172f1c as hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/697cd6a363e514bff0d4ec954fc4034b/A/9fe9bcf9a3c74206a24135894d172f1c 2024-11-16T10:00:15,694 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/697cd6a363e514bff0d4ec954fc4034b/A/9fe9bcf9a3c74206a24135894d172f1c, entries=150, sequenceid=435, filesize=12.0 K 2024-11-16T10:00:15,695 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/697cd6a363e514bff0d4ec954fc4034b/.tmp/B/8262e61da0be43eaacaeb6971c33d6ad as hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/697cd6a363e514bff0d4ec954fc4034b/B/8262e61da0be43eaacaeb6971c33d6ad 2024-11-16T10:00:15,697 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/697cd6a363e514bff0d4ec954fc4034b/B/8262e61da0be43eaacaeb6971c33d6ad, entries=150, sequenceid=435, filesize=12.0 K 2024-11-16T10:00:15,698 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/697cd6a363e514bff0d4ec954fc4034b/.tmp/C/add9632fbd004800979d682fb30d05a0 as hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/697cd6a363e514bff0d4ec954fc4034b/C/add9632fbd004800979d682fb30d05a0 2024-11-16T10:00:15,700 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/697cd6a363e514bff0d4ec954fc4034b/C/add9632fbd004800979d682fb30d05a0, entries=150, sequenceid=435, filesize=12.0 K 2024-11-16T10:00:15,701 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~167.72 KB/171750, heapSize ~440.16 KB/450720, currentSize=26.84 KB/27480 for 697cd6a363e514bff0d4ec954fc4034b in 1237ms, sequenceid=435, compaction requested=true 2024-11-16T10:00:15,701 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 697cd6a363e514bff0d4ec954fc4034b: 2024-11-16T10:00:15,701 INFO [RS_CLOSE_REGION-regionserver/037c58a19ddb:0-0 {event_type=M_RS_CLOSE_REGION, pid=140}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1731751192181.697cd6a363e514bff0d4ec954fc4034b. 2024-11-16T10:00:15,701 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 697cd6a363e514bff0d4ec954fc4034b:A, priority=-2147483648, current under compaction store size is 1 2024-11-16T10:00:15,701 DEBUG [RS_CLOSE_REGION-regionserver/037c58a19ddb:0-0 {event_type=M_RS_CLOSE_REGION, pid=140}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1731751192181.697cd6a363e514bff0d4ec954fc4034b. 2024-11-16T10:00:15,701 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-16T10:00:15,701 DEBUG [RS_CLOSE_REGION-regionserver/037c58a19ddb:0-0 {event_type=M_RS_CLOSE_REGION, pid=140}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1731751192181.697cd6a363e514bff0d4ec954fc4034b. after waiting 0 ms 2024-11-16T10:00:15,701 DEBUG [RS:0;037c58a19ddb:37267-shortCompactions-0 {}] regionserver.CompactSplit(450): Not compacting TestAcidGuarantees,,1731751192181.697cd6a363e514bff0d4ec954fc4034b. because compaction request was cancelled 2024-11-16T10:00:15,701 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 697cd6a363e514bff0d4ec954fc4034b:B, priority=-2147483648, current under compaction store size is 2 2024-11-16T10:00:15,701 DEBUG [RS_CLOSE_REGION-regionserver/037c58a19ddb:0-0 {event_type=M_RS_CLOSE_REGION, pid=140}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1731751192181.697cd6a363e514bff0d4ec954fc4034b. 2024-11-16T10:00:15,701 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-16T10:00:15,701 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 697cd6a363e514bff0d4ec954fc4034b:C, priority=-2147483648, current under compaction store size is 3 2024-11-16T10:00:15,701 DEBUG [RS:0;037c58a19ddb:37267-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 697cd6a363e514bff0d4ec954fc4034b:A 2024-11-16T10:00:15,701 DEBUG [RS:0;037c58a19ddb:37267-longCompactions-0 {}] regionserver.CompactSplit(450): Not compacting TestAcidGuarantees,,1731751192181.697cd6a363e514bff0d4ec954fc4034b. because compaction request was cancelled 2024-11-16T10:00:15,701 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-16T10:00:15,701 DEBUG [RS:0;037c58a19ddb:37267-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 697cd6a363e514bff0d4ec954fc4034b:B 2024-11-16T10:00:15,701 DEBUG [RS:0;037c58a19ddb:37267-shortCompactions-0 {}] regionserver.CompactSplit(450): Not compacting TestAcidGuarantees,,1731751192181.697cd6a363e514bff0d4ec954fc4034b. because compaction request was cancelled 2024-11-16T10:00:15,701 DEBUG [RS:0;037c58a19ddb:37267-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 697cd6a363e514bff0d4ec954fc4034b:C 2024-11-16T10:00:15,701 INFO [RS_CLOSE_REGION-regionserver/037c58a19ddb:0-0 {event_type=M_RS_CLOSE_REGION, pid=140}] regionserver.HRegion(2902): Flushing 697cd6a363e514bff0d4ec954fc4034b 3/3 column families, dataSize=26.84 KB heapSize=71.06 KB 2024-11-16T10:00:15,704 DEBUG [RS_CLOSE_REGION-regionserver/037c58a19ddb:0-0 {event_type=M_RS_CLOSE_REGION, pid=140}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/697cd6a363e514bff0d4ec954fc4034b/.tmp/A/feebd758ab0e40b086db26d4446b7716 is 50, key is test_row_0/A:col10/1731751214473/Put/seqid=0 2024-11-16T10:00:15,707 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37591 is added to blk_1073742361_1537 (size=9857) 2024-11-16T10:00:16,108 INFO [RS_CLOSE_REGION-regionserver/037c58a19ddb:0-0 {event_type=M_RS_CLOSE_REGION, pid=140}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=8.95 KB at sequenceid=442 (bloomFilter=true), to=hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/697cd6a363e514bff0d4ec954fc4034b/.tmp/A/feebd758ab0e40b086db26d4446b7716 2024-11-16T10:00:16,113 DEBUG [RS_CLOSE_REGION-regionserver/037c58a19ddb:0-0 {event_type=M_RS_CLOSE_REGION, pid=140}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/697cd6a363e514bff0d4ec954fc4034b/.tmp/B/07c17bac7c0644d5965d02c99e4723ef is 50, key is test_row_0/B:col10/1731751214473/Put/seqid=0 2024-11-16T10:00:16,116 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37591 is added to blk_1073742362_1538 (size=9857) 2024-11-16T10:00:16,183 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37877 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=137 2024-11-16T10:00:16,516 INFO [RS_CLOSE_REGION-regionserver/037c58a19ddb:0-0 {event_type=M_RS_CLOSE_REGION, pid=140}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=8.95 KB at sequenceid=442 (bloomFilter=true), to=hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/697cd6a363e514bff0d4ec954fc4034b/.tmp/B/07c17bac7c0644d5965d02c99e4723ef 2024-11-16T10:00:16,521 DEBUG [RS_CLOSE_REGION-regionserver/037c58a19ddb:0-0 {event_type=M_RS_CLOSE_REGION, pid=140}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/697cd6a363e514bff0d4ec954fc4034b/.tmp/C/e579817550f14a259b993196a4a5c56d is 50, key is test_row_0/C:col10/1731751214473/Put/seqid=0 2024-11-16T10:00:16,524 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37591 is added to blk_1073742363_1539 (size=9857) 2024-11-16T10:00:16,925 INFO [RS_CLOSE_REGION-regionserver/037c58a19ddb:0-0 {event_type=M_RS_CLOSE_REGION, pid=140}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=8.95 KB at sequenceid=442 (bloomFilter=true), to=hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/697cd6a363e514bff0d4ec954fc4034b/.tmp/C/e579817550f14a259b993196a4a5c56d 2024-11-16T10:00:16,929 DEBUG [RS_CLOSE_REGION-regionserver/037c58a19ddb:0-0 {event_type=M_RS_CLOSE_REGION, pid=140}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/697cd6a363e514bff0d4ec954fc4034b/.tmp/A/feebd758ab0e40b086db26d4446b7716 as hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/697cd6a363e514bff0d4ec954fc4034b/A/feebd758ab0e40b086db26d4446b7716 2024-11-16T10:00:16,931 INFO [RS_CLOSE_REGION-regionserver/037c58a19ddb:0-0 {event_type=M_RS_CLOSE_REGION, pid=140}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/697cd6a363e514bff0d4ec954fc4034b/A/feebd758ab0e40b086db26d4446b7716, entries=100, sequenceid=442, filesize=9.6 K 2024-11-16T10:00:16,932 DEBUG [RS_CLOSE_REGION-regionserver/037c58a19ddb:0-0 {event_type=M_RS_CLOSE_REGION, pid=140}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/697cd6a363e514bff0d4ec954fc4034b/.tmp/B/07c17bac7c0644d5965d02c99e4723ef as hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/697cd6a363e514bff0d4ec954fc4034b/B/07c17bac7c0644d5965d02c99e4723ef 2024-11-16T10:00:16,934 INFO [RS_CLOSE_REGION-regionserver/037c58a19ddb:0-0 {event_type=M_RS_CLOSE_REGION, pid=140}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/697cd6a363e514bff0d4ec954fc4034b/B/07c17bac7c0644d5965d02c99e4723ef, entries=100, sequenceid=442, filesize=9.6 K 2024-11-16T10:00:16,935 DEBUG [RS_CLOSE_REGION-regionserver/037c58a19ddb:0-0 {event_type=M_RS_CLOSE_REGION, pid=140}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/697cd6a363e514bff0d4ec954fc4034b/.tmp/C/e579817550f14a259b993196a4a5c56d as hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/697cd6a363e514bff0d4ec954fc4034b/C/e579817550f14a259b993196a4a5c56d 2024-11-16T10:00:16,937 INFO [RS_CLOSE_REGION-regionserver/037c58a19ddb:0-0 {event_type=M_RS_CLOSE_REGION, pid=140}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/697cd6a363e514bff0d4ec954fc4034b/C/e579817550f14a259b993196a4a5c56d, entries=100, sequenceid=442, filesize=9.6 K 2024-11-16T10:00:16,938 INFO [RS_CLOSE_REGION-regionserver/037c58a19ddb:0-0 {event_type=M_RS_CLOSE_REGION, pid=140}] regionserver.HRegion(3140): Finished flush of dataSize ~26.84 KB/27480, heapSize ~71.02 KB/72720, currentSize=0 B/0 for 697cd6a363e514bff0d4ec954fc4034b in 1237ms, sequenceid=442, compaction requested=true 2024-11-16T10:00:16,938 DEBUG [StoreCloser-TestAcidGuarantees,,1731751192181.697cd6a363e514bff0d4ec954fc4034b.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/697cd6a363e514bff0d4ec954fc4034b/A/b0882ffe2ec9422087f061171d91e3bc, hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/697cd6a363e514bff0d4ec954fc4034b/A/93f09bcb15ca453ba30359031b06d38d, hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/697cd6a363e514bff0d4ec954fc4034b/A/601acfc22f994b6ea38aa89aebc0dcc2, hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/697cd6a363e514bff0d4ec954fc4034b/A/0e15f225c9fc496c9feeb6c7cfeaf92b, hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/697cd6a363e514bff0d4ec954fc4034b/A/47804894c124483281dc4ec91f40d7a4, hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/697cd6a363e514bff0d4ec954fc4034b/A/75216ae3e27a4a6fa6d1f190faf54871, hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/697cd6a363e514bff0d4ec954fc4034b/A/387791a6a1904f44b3973f11b149c72a, hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/697cd6a363e514bff0d4ec954fc4034b/A/0572be26924348a1ba9c6d41e1d2fc5f, hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/697cd6a363e514bff0d4ec954fc4034b/A/6fd014921977453799c306ea60a1a1ef, hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/697cd6a363e514bff0d4ec954fc4034b/A/aa422c222ac1447d8325f707092eaf7e, hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/697cd6a363e514bff0d4ec954fc4034b/A/3cecc1d82dd343439dbccdd090ece5fd, hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/697cd6a363e514bff0d4ec954fc4034b/A/5b4e2ff342e54bdc84037fe744487a35, hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/697cd6a363e514bff0d4ec954fc4034b/A/86ab58c9d50f47258676a412736ef737, hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/697cd6a363e514bff0d4ec954fc4034b/A/7795a15b11b2402dbd7704ab7054d526, hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/697cd6a363e514bff0d4ec954fc4034b/A/a0ee859be0c44f4b8e62931f139ef719, hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/697cd6a363e514bff0d4ec954fc4034b/A/844f11e79e4445f49c53f85d0e276e2d, hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/697cd6a363e514bff0d4ec954fc4034b/A/c3c82a2503ed413f9db2739f818730bf, hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/697cd6a363e514bff0d4ec954fc4034b/A/56afeb41771c4583b16aa0eb393faa0a, hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/697cd6a363e514bff0d4ec954fc4034b/A/25d58730a74d47b9ae42efbe06a1c749, hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/697cd6a363e514bff0d4ec954fc4034b/A/6287c88120694b81afbdd7d41cdc78ff, hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/697cd6a363e514bff0d4ec954fc4034b/A/cdb34818a22c4599969803da6533b889, hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/697cd6a363e514bff0d4ec954fc4034b/A/ca8e96abf12a4adfb357b4e54e388818] to archive 2024-11-16T10:00:16,939 DEBUG [StoreCloser-TestAcidGuarantees,,1731751192181.697cd6a363e514bff0d4ec954fc4034b.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-16T10:00:16,940 DEBUG [StoreCloser-TestAcidGuarantees,,1731751192181.697cd6a363e514bff0d4ec954fc4034b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/697cd6a363e514bff0d4ec954fc4034b/A/b0882ffe2ec9422087f061171d91e3bc to hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/archive/data/default/TestAcidGuarantees/697cd6a363e514bff0d4ec954fc4034b/A/b0882ffe2ec9422087f061171d91e3bc 2024-11-16T10:00:16,941 DEBUG [StoreCloser-TestAcidGuarantees,,1731751192181.697cd6a363e514bff0d4ec954fc4034b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/697cd6a363e514bff0d4ec954fc4034b/A/93f09bcb15ca453ba30359031b06d38d to hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/archive/data/default/TestAcidGuarantees/697cd6a363e514bff0d4ec954fc4034b/A/93f09bcb15ca453ba30359031b06d38d 2024-11-16T10:00:16,942 DEBUG [StoreCloser-TestAcidGuarantees,,1731751192181.697cd6a363e514bff0d4ec954fc4034b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/697cd6a363e514bff0d4ec954fc4034b/A/601acfc22f994b6ea38aa89aebc0dcc2 to hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/archive/data/default/TestAcidGuarantees/697cd6a363e514bff0d4ec954fc4034b/A/601acfc22f994b6ea38aa89aebc0dcc2 2024-11-16T10:00:16,942 DEBUG [StoreCloser-TestAcidGuarantees,,1731751192181.697cd6a363e514bff0d4ec954fc4034b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/697cd6a363e514bff0d4ec954fc4034b/A/0e15f225c9fc496c9feeb6c7cfeaf92b to hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/archive/data/default/TestAcidGuarantees/697cd6a363e514bff0d4ec954fc4034b/A/0e15f225c9fc496c9feeb6c7cfeaf92b 2024-11-16T10:00:16,943 DEBUG [StoreCloser-TestAcidGuarantees,,1731751192181.697cd6a363e514bff0d4ec954fc4034b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/697cd6a363e514bff0d4ec954fc4034b/A/47804894c124483281dc4ec91f40d7a4 to hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/archive/data/default/TestAcidGuarantees/697cd6a363e514bff0d4ec954fc4034b/A/47804894c124483281dc4ec91f40d7a4 2024-11-16T10:00:16,944 DEBUG [StoreCloser-TestAcidGuarantees,,1731751192181.697cd6a363e514bff0d4ec954fc4034b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/697cd6a363e514bff0d4ec954fc4034b/A/75216ae3e27a4a6fa6d1f190faf54871 to hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/archive/data/default/TestAcidGuarantees/697cd6a363e514bff0d4ec954fc4034b/A/75216ae3e27a4a6fa6d1f190faf54871 2024-11-16T10:00:16,945 DEBUG [StoreCloser-TestAcidGuarantees,,1731751192181.697cd6a363e514bff0d4ec954fc4034b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/697cd6a363e514bff0d4ec954fc4034b/A/387791a6a1904f44b3973f11b149c72a to hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/archive/data/default/TestAcidGuarantees/697cd6a363e514bff0d4ec954fc4034b/A/387791a6a1904f44b3973f11b149c72a 2024-11-16T10:00:16,945 DEBUG [StoreCloser-TestAcidGuarantees,,1731751192181.697cd6a363e514bff0d4ec954fc4034b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/697cd6a363e514bff0d4ec954fc4034b/A/0572be26924348a1ba9c6d41e1d2fc5f to hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/archive/data/default/TestAcidGuarantees/697cd6a363e514bff0d4ec954fc4034b/A/0572be26924348a1ba9c6d41e1d2fc5f 2024-11-16T10:00:16,946 DEBUG [StoreCloser-TestAcidGuarantees,,1731751192181.697cd6a363e514bff0d4ec954fc4034b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/697cd6a363e514bff0d4ec954fc4034b/A/6fd014921977453799c306ea60a1a1ef to hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/archive/data/default/TestAcidGuarantees/697cd6a363e514bff0d4ec954fc4034b/A/6fd014921977453799c306ea60a1a1ef 2024-11-16T10:00:16,947 DEBUG [StoreCloser-TestAcidGuarantees,,1731751192181.697cd6a363e514bff0d4ec954fc4034b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/697cd6a363e514bff0d4ec954fc4034b/A/aa422c222ac1447d8325f707092eaf7e to hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/archive/data/default/TestAcidGuarantees/697cd6a363e514bff0d4ec954fc4034b/A/aa422c222ac1447d8325f707092eaf7e 2024-11-16T10:00:16,947 DEBUG [StoreCloser-TestAcidGuarantees,,1731751192181.697cd6a363e514bff0d4ec954fc4034b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/697cd6a363e514bff0d4ec954fc4034b/A/3cecc1d82dd343439dbccdd090ece5fd to hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/archive/data/default/TestAcidGuarantees/697cd6a363e514bff0d4ec954fc4034b/A/3cecc1d82dd343439dbccdd090ece5fd 2024-11-16T10:00:16,948 DEBUG [StoreCloser-TestAcidGuarantees,,1731751192181.697cd6a363e514bff0d4ec954fc4034b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/697cd6a363e514bff0d4ec954fc4034b/A/5b4e2ff342e54bdc84037fe744487a35 to hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/archive/data/default/TestAcidGuarantees/697cd6a363e514bff0d4ec954fc4034b/A/5b4e2ff342e54bdc84037fe744487a35 2024-11-16T10:00:16,949 DEBUG [StoreCloser-TestAcidGuarantees,,1731751192181.697cd6a363e514bff0d4ec954fc4034b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/697cd6a363e514bff0d4ec954fc4034b/A/86ab58c9d50f47258676a412736ef737 to hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/archive/data/default/TestAcidGuarantees/697cd6a363e514bff0d4ec954fc4034b/A/86ab58c9d50f47258676a412736ef737 2024-11-16T10:00:16,950 DEBUG [StoreCloser-TestAcidGuarantees,,1731751192181.697cd6a363e514bff0d4ec954fc4034b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/697cd6a363e514bff0d4ec954fc4034b/A/7795a15b11b2402dbd7704ab7054d526 to hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/archive/data/default/TestAcidGuarantees/697cd6a363e514bff0d4ec954fc4034b/A/7795a15b11b2402dbd7704ab7054d526 2024-11-16T10:00:16,950 DEBUG [StoreCloser-TestAcidGuarantees,,1731751192181.697cd6a363e514bff0d4ec954fc4034b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/697cd6a363e514bff0d4ec954fc4034b/A/a0ee859be0c44f4b8e62931f139ef719 to hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/archive/data/default/TestAcidGuarantees/697cd6a363e514bff0d4ec954fc4034b/A/a0ee859be0c44f4b8e62931f139ef719 2024-11-16T10:00:16,951 DEBUG [StoreCloser-TestAcidGuarantees,,1731751192181.697cd6a363e514bff0d4ec954fc4034b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/697cd6a363e514bff0d4ec954fc4034b/A/844f11e79e4445f49c53f85d0e276e2d to hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/archive/data/default/TestAcidGuarantees/697cd6a363e514bff0d4ec954fc4034b/A/844f11e79e4445f49c53f85d0e276e2d 2024-11-16T10:00:16,952 DEBUG [StoreCloser-TestAcidGuarantees,,1731751192181.697cd6a363e514bff0d4ec954fc4034b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/697cd6a363e514bff0d4ec954fc4034b/A/c3c82a2503ed413f9db2739f818730bf to hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/archive/data/default/TestAcidGuarantees/697cd6a363e514bff0d4ec954fc4034b/A/c3c82a2503ed413f9db2739f818730bf 2024-11-16T10:00:16,953 DEBUG [StoreCloser-TestAcidGuarantees,,1731751192181.697cd6a363e514bff0d4ec954fc4034b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/697cd6a363e514bff0d4ec954fc4034b/A/56afeb41771c4583b16aa0eb393faa0a to hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/archive/data/default/TestAcidGuarantees/697cd6a363e514bff0d4ec954fc4034b/A/56afeb41771c4583b16aa0eb393faa0a 2024-11-16T10:00:16,954 DEBUG [StoreCloser-TestAcidGuarantees,,1731751192181.697cd6a363e514bff0d4ec954fc4034b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/697cd6a363e514bff0d4ec954fc4034b/A/25d58730a74d47b9ae42efbe06a1c749 to hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/archive/data/default/TestAcidGuarantees/697cd6a363e514bff0d4ec954fc4034b/A/25d58730a74d47b9ae42efbe06a1c749 2024-11-16T10:00:16,955 DEBUG [StoreCloser-TestAcidGuarantees,,1731751192181.697cd6a363e514bff0d4ec954fc4034b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/697cd6a363e514bff0d4ec954fc4034b/A/6287c88120694b81afbdd7d41cdc78ff to hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/archive/data/default/TestAcidGuarantees/697cd6a363e514bff0d4ec954fc4034b/A/6287c88120694b81afbdd7d41cdc78ff 2024-11-16T10:00:16,956 DEBUG [StoreCloser-TestAcidGuarantees,,1731751192181.697cd6a363e514bff0d4ec954fc4034b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/697cd6a363e514bff0d4ec954fc4034b/A/cdb34818a22c4599969803da6533b889 to hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/archive/data/default/TestAcidGuarantees/697cd6a363e514bff0d4ec954fc4034b/A/cdb34818a22c4599969803da6533b889 2024-11-16T10:00:16,956 DEBUG [StoreCloser-TestAcidGuarantees,,1731751192181.697cd6a363e514bff0d4ec954fc4034b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/697cd6a363e514bff0d4ec954fc4034b/A/ca8e96abf12a4adfb357b4e54e388818 to hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/archive/data/default/TestAcidGuarantees/697cd6a363e514bff0d4ec954fc4034b/A/ca8e96abf12a4adfb357b4e54e388818 2024-11-16T10:00:16,958 DEBUG [StoreCloser-TestAcidGuarantees,,1731751192181.697cd6a363e514bff0d4ec954fc4034b.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/697cd6a363e514bff0d4ec954fc4034b/B/9f94221481804f42b846e9e9614916f6, hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/697cd6a363e514bff0d4ec954fc4034b/B/34019d6dca5944bb901ff94f64922d8a, hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/697cd6a363e514bff0d4ec954fc4034b/B/edb64d88ad5945048c2fa5d8962ee677, hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/697cd6a363e514bff0d4ec954fc4034b/B/380b382ef5cb45f595d4ec3ad8d26436, hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/697cd6a363e514bff0d4ec954fc4034b/B/1d367eb7c0064329aaffa8b4aaa027d7, hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/697cd6a363e514bff0d4ec954fc4034b/B/173c5e49d87a4489abab8fc2e7bdade3, hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/697cd6a363e514bff0d4ec954fc4034b/B/127fba5983df4cef875b7429842baf4e, hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/697cd6a363e514bff0d4ec954fc4034b/B/0a22ef23afe44699a9b457690504a95f, hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/697cd6a363e514bff0d4ec954fc4034b/B/090ecb975fe84165b550dd5f4def4066, hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/697cd6a363e514bff0d4ec954fc4034b/B/5ebf14cd6b1c424d9087c461c3fefa86, hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/697cd6a363e514bff0d4ec954fc4034b/B/df81870aa751418da08296e961c67ac3, hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/697cd6a363e514bff0d4ec954fc4034b/B/8048b8cfb710419bb4932c2b21933a5c, hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/697cd6a363e514bff0d4ec954fc4034b/B/d6f9eea4334549a18e19ffa0b74c53ac, hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/697cd6a363e514bff0d4ec954fc4034b/B/a2aa066446a24730ae0703b47b429d8d, hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/697cd6a363e514bff0d4ec954fc4034b/B/a70bf22e5f60433089494a2d1988999b, hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/697cd6a363e514bff0d4ec954fc4034b/B/6cdb974236134267a7e4d6f4f42abaa2, hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/697cd6a363e514bff0d4ec954fc4034b/B/3e7b206106e64728ae5589c7ff779a38, hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/697cd6a363e514bff0d4ec954fc4034b/B/c5f885ecf9af4420be16cd8dd42d29cf, hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/697cd6a363e514bff0d4ec954fc4034b/B/376083493aa8414fbc47f4cca9482837, hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/697cd6a363e514bff0d4ec954fc4034b/B/afcabd057a004b74bc5fe9707dc6c2ca, hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/697cd6a363e514bff0d4ec954fc4034b/B/743c3e50c2dc453da9f17314fa21a38a, hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/697cd6a363e514bff0d4ec954fc4034b/B/00e2727159444e7b88ef9c3ed5a91da2] to archive 2024-11-16T10:00:16,958 DEBUG [StoreCloser-TestAcidGuarantees,,1731751192181.697cd6a363e514bff0d4ec954fc4034b.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-16T10:00:16,959 DEBUG [StoreCloser-TestAcidGuarantees,,1731751192181.697cd6a363e514bff0d4ec954fc4034b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/697cd6a363e514bff0d4ec954fc4034b/B/9f94221481804f42b846e9e9614916f6 to hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/archive/data/default/TestAcidGuarantees/697cd6a363e514bff0d4ec954fc4034b/B/9f94221481804f42b846e9e9614916f6 2024-11-16T10:00:16,960 DEBUG [StoreCloser-TestAcidGuarantees,,1731751192181.697cd6a363e514bff0d4ec954fc4034b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/697cd6a363e514bff0d4ec954fc4034b/B/34019d6dca5944bb901ff94f64922d8a to hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/archive/data/default/TestAcidGuarantees/697cd6a363e514bff0d4ec954fc4034b/B/34019d6dca5944bb901ff94f64922d8a 2024-11-16T10:00:16,961 DEBUG [StoreCloser-TestAcidGuarantees,,1731751192181.697cd6a363e514bff0d4ec954fc4034b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/697cd6a363e514bff0d4ec954fc4034b/B/edb64d88ad5945048c2fa5d8962ee677 to hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/archive/data/default/TestAcidGuarantees/697cd6a363e514bff0d4ec954fc4034b/B/edb64d88ad5945048c2fa5d8962ee677 2024-11-16T10:00:16,962 DEBUG [StoreCloser-TestAcidGuarantees,,1731751192181.697cd6a363e514bff0d4ec954fc4034b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/697cd6a363e514bff0d4ec954fc4034b/B/380b382ef5cb45f595d4ec3ad8d26436 to hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/archive/data/default/TestAcidGuarantees/697cd6a363e514bff0d4ec954fc4034b/B/380b382ef5cb45f595d4ec3ad8d26436 2024-11-16T10:00:16,963 DEBUG [StoreCloser-TestAcidGuarantees,,1731751192181.697cd6a363e514bff0d4ec954fc4034b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/697cd6a363e514bff0d4ec954fc4034b/B/1d367eb7c0064329aaffa8b4aaa027d7 to hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/archive/data/default/TestAcidGuarantees/697cd6a363e514bff0d4ec954fc4034b/B/1d367eb7c0064329aaffa8b4aaa027d7 2024-11-16T10:00:16,964 DEBUG [StoreCloser-TestAcidGuarantees,,1731751192181.697cd6a363e514bff0d4ec954fc4034b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/697cd6a363e514bff0d4ec954fc4034b/B/173c5e49d87a4489abab8fc2e7bdade3 to hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/archive/data/default/TestAcidGuarantees/697cd6a363e514bff0d4ec954fc4034b/B/173c5e49d87a4489abab8fc2e7bdade3 2024-11-16T10:00:16,964 DEBUG [StoreCloser-TestAcidGuarantees,,1731751192181.697cd6a363e514bff0d4ec954fc4034b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/697cd6a363e514bff0d4ec954fc4034b/B/127fba5983df4cef875b7429842baf4e to hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/archive/data/default/TestAcidGuarantees/697cd6a363e514bff0d4ec954fc4034b/B/127fba5983df4cef875b7429842baf4e 2024-11-16T10:00:16,965 DEBUG [StoreCloser-TestAcidGuarantees,,1731751192181.697cd6a363e514bff0d4ec954fc4034b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/697cd6a363e514bff0d4ec954fc4034b/B/0a22ef23afe44699a9b457690504a95f to hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/archive/data/default/TestAcidGuarantees/697cd6a363e514bff0d4ec954fc4034b/B/0a22ef23afe44699a9b457690504a95f 2024-11-16T10:00:16,966 DEBUG [StoreCloser-TestAcidGuarantees,,1731751192181.697cd6a363e514bff0d4ec954fc4034b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/697cd6a363e514bff0d4ec954fc4034b/B/090ecb975fe84165b550dd5f4def4066 to hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/archive/data/default/TestAcidGuarantees/697cd6a363e514bff0d4ec954fc4034b/B/090ecb975fe84165b550dd5f4def4066 2024-11-16T10:00:16,967 DEBUG [StoreCloser-TestAcidGuarantees,,1731751192181.697cd6a363e514bff0d4ec954fc4034b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/697cd6a363e514bff0d4ec954fc4034b/B/5ebf14cd6b1c424d9087c461c3fefa86 to hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/archive/data/default/TestAcidGuarantees/697cd6a363e514bff0d4ec954fc4034b/B/5ebf14cd6b1c424d9087c461c3fefa86 2024-11-16T10:00:16,967 DEBUG [StoreCloser-TestAcidGuarantees,,1731751192181.697cd6a363e514bff0d4ec954fc4034b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/697cd6a363e514bff0d4ec954fc4034b/B/df81870aa751418da08296e961c67ac3 to hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/archive/data/default/TestAcidGuarantees/697cd6a363e514bff0d4ec954fc4034b/B/df81870aa751418da08296e961c67ac3 2024-11-16T10:00:16,968 DEBUG [StoreCloser-TestAcidGuarantees,,1731751192181.697cd6a363e514bff0d4ec954fc4034b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/697cd6a363e514bff0d4ec954fc4034b/B/8048b8cfb710419bb4932c2b21933a5c to hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/archive/data/default/TestAcidGuarantees/697cd6a363e514bff0d4ec954fc4034b/B/8048b8cfb710419bb4932c2b21933a5c 2024-11-16T10:00:16,969 DEBUG [StoreCloser-TestAcidGuarantees,,1731751192181.697cd6a363e514bff0d4ec954fc4034b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/697cd6a363e514bff0d4ec954fc4034b/B/d6f9eea4334549a18e19ffa0b74c53ac to hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/archive/data/default/TestAcidGuarantees/697cd6a363e514bff0d4ec954fc4034b/B/d6f9eea4334549a18e19ffa0b74c53ac 2024-11-16T10:00:16,970 DEBUG [StoreCloser-TestAcidGuarantees,,1731751192181.697cd6a363e514bff0d4ec954fc4034b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/697cd6a363e514bff0d4ec954fc4034b/B/a2aa066446a24730ae0703b47b429d8d to hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/archive/data/default/TestAcidGuarantees/697cd6a363e514bff0d4ec954fc4034b/B/a2aa066446a24730ae0703b47b429d8d 2024-11-16T10:00:16,970 DEBUG [StoreCloser-TestAcidGuarantees,,1731751192181.697cd6a363e514bff0d4ec954fc4034b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/697cd6a363e514bff0d4ec954fc4034b/B/a70bf22e5f60433089494a2d1988999b to hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/archive/data/default/TestAcidGuarantees/697cd6a363e514bff0d4ec954fc4034b/B/a70bf22e5f60433089494a2d1988999b 2024-11-16T10:00:16,971 DEBUG [StoreCloser-TestAcidGuarantees,,1731751192181.697cd6a363e514bff0d4ec954fc4034b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/697cd6a363e514bff0d4ec954fc4034b/B/6cdb974236134267a7e4d6f4f42abaa2 to hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/archive/data/default/TestAcidGuarantees/697cd6a363e514bff0d4ec954fc4034b/B/6cdb974236134267a7e4d6f4f42abaa2 2024-11-16T10:00:16,972 DEBUG [StoreCloser-TestAcidGuarantees,,1731751192181.697cd6a363e514bff0d4ec954fc4034b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/697cd6a363e514bff0d4ec954fc4034b/B/3e7b206106e64728ae5589c7ff779a38 to hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/archive/data/default/TestAcidGuarantees/697cd6a363e514bff0d4ec954fc4034b/B/3e7b206106e64728ae5589c7ff779a38 2024-11-16T10:00:16,973 DEBUG [StoreCloser-TestAcidGuarantees,,1731751192181.697cd6a363e514bff0d4ec954fc4034b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/697cd6a363e514bff0d4ec954fc4034b/B/c5f885ecf9af4420be16cd8dd42d29cf to hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/archive/data/default/TestAcidGuarantees/697cd6a363e514bff0d4ec954fc4034b/B/c5f885ecf9af4420be16cd8dd42d29cf 2024-11-16T10:00:16,973 DEBUG [StoreCloser-TestAcidGuarantees,,1731751192181.697cd6a363e514bff0d4ec954fc4034b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/697cd6a363e514bff0d4ec954fc4034b/B/376083493aa8414fbc47f4cca9482837 to hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/archive/data/default/TestAcidGuarantees/697cd6a363e514bff0d4ec954fc4034b/B/376083493aa8414fbc47f4cca9482837 2024-11-16T10:00:16,974 DEBUG [StoreCloser-TestAcidGuarantees,,1731751192181.697cd6a363e514bff0d4ec954fc4034b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/697cd6a363e514bff0d4ec954fc4034b/B/afcabd057a004b74bc5fe9707dc6c2ca to hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/archive/data/default/TestAcidGuarantees/697cd6a363e514bff0d4ec954fc4034b/B/afcabd057a004b74bc5fe9707dc6c2ca 2024-11-16T10:00:16,975 DEBUG [StoreCloser-TestAcidGuarantees,,1731751192181.697cd6a363e514bff0d4ec954fc4034b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/697cd6a363e514bff0d4ec954fc4034b/B/743c3e50c2dc453da9f17314fa21a38a to hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/archive/data/default/TestAcidGuarantees/697cd6a363e514bff0d4ec954fc4034b/B/743c3e50c2dc453da9f17314fa21a38a 2024-11-16T10:00:16,976 DEBUG [StoreCloser-TestAcidGuarantees,,1731751192181.697cd6a363e514bff0d4ec954fc4034b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/697cd6a363e514bff0d4ec954fc4034b/B/00e2727159444e7b88ef9c3ed5a91da2 to hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/archive/data/default/TestAcidGuarantees/697cd6a363e514bff0d4ec954fc4034b/B/00e2727159444e7b88ef9c3ed5a91da2 2024-11-16T10:00:16,977 DEBUG [StoreCloser-TestAcidGuarantees,,1731751192181.697cd6a363e514bff0d4ec954fc4034b.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/697cd6a363e514bff0d4ec954fc4034b/C/1ca86ee061304d5baf51f01fb8c60b7b, hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/697cd6a363e514bff0d4ec954fc4034b/C/e2fa6223e0924717854b53c074fd39fd, hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/697cd6a363e514bff0d4ec954fc4034b/C/336e6611f01c42eca208c40d0f7121de, hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/697cd6a363e514bff0d4ec954fc4034b/C/9b994a5406404b3fa89fea7191ed65e7, hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/697cd6a363e514bff0d4ec954fc4034b/C/e4b523607a6e49e8a22c3232d2fa54ec, hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/697cd6a363e514bff0d4ec954fc4034b/C/1eda3944ab0448908f50204d73526559, hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/697cd6a363e514bff0d4ec954fc4034b/C/fd7057e35b004c38881704f7e925903e, hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/697cd6a363e514bff0d4ec954fc4034b/C/db2e5ea5d8c048eaa89b7e96a9c2e23a, hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/697cd6a363e514bff0d4ec954fc4034b/C/cb29188c3e284c5aafb352c461876405, hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/697cd6a363e514bff0d4ec954fc4034b/C/4c12439cbf7a45158cddfb8f850956bf, hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/697cd6a363e514bff0d4ec954fc4034b/C/1a1514cb66b8456bab35a7ce294b1663, hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/697cd6a363e514bff0d4ec954fc4034b/C/699da832fda74291b53ee546204927b7, hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/697cd6a363e514bff0d4ec954fc4034b/C/8afbac04ec764dc4b9dbddce6d7afb77, hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/697cd6a363e514bff0d4ec954fc4034b/C/22cb10ec5fd249f3ba682ac831965d6d, hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/697cd6a363e514bff0d4ec954fc4034b/C/475bfdd4b2ed4ab9b2d86b58ceb856f3, hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/697cd6a363e514bff0d4ec954fc4034b/C/6846a39ce2564b2cbeec112bc41ff608, hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/697cd6a363e514bff0d4ec954fc4034b/C/1d64ef6ef6ef4156812f479b5da065f6, hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/697cd6a363e514bff0d4ec954fc4034b/C/9237610090d64e16b429d0a108aa1d65, hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/697cd6a363e514bff0d4ec954fc4034b/C/0c90ee06996748789e1d25093a7f1e25, hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/697cd6a363e514bff0d4ec954fc4034b/C/0a8b4857d6014068ba12683461bf5532, hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/697cd6a363e514bff0d4ec954fc4034b/C/e4d5f1ee2bca43e5adfbd16e2b3aa12a, hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/697cd6a363e514bff0d4ec954fc4034b/C/2b55b0ed5f8942d1a3f6a2eb0a32fc72] to archive 2024-11-16T10:00:16,977 DEBUG [StoreCloser-TestAcidGuarantees,,1731751192181.697cd6a363e514bff0d4ec954fc4034b.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-16T10:00:16,979 DEBUG [StoreCloser-TestAcidGuarantees,,1731751192181.697cd6a363e514bff0d4ec954fc4034b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/697cd6a363e514bff0d4ec954fc4034b/C/1ca86ee061304d5baf51f01fb8c60b7b to hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/archive/data/default/TestAcidGuarantees/697cd6a363e514bff0d4ec954fc4034b/C/1ca86ee061304d5baf51f01fb8c60b7b 2024-11-16T10:00:16,980 DEBUG [StoreCloser-TestAcidGuarantees,,1731751192181.697cd6a363e514bff0d4ec954fc4034b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/697cd6a363e514bff0d4ec954fc4034b/C/e2fa6223e0924717854b53c074fd39fd to hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/archive/data/default/TestAcidGuarantees/697cd6a363e514bff0d4ec954fc4034b/C/e2fa6223e0924717854b53c074fd39fd 2024-11-16T10:00:16,980 DEBUG [StoreCloser-TestAcidGuarantees,,1731751192181.697cd6a363e514bff0d4ec954fc4034b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/697cd6a363e514bff0d4ec954fc4034b/C/336e6611f01c42eca208c40d0f7121de to hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/archive/data/default/TestAcidGuarantees/697cd6a363e514bff0d4ec954fc4034b/C/336e6611f01c42eca208c40d0f7121de 2024-11-16T10:00:16,981 DEBUG [StoreCloser-TestAcidGuarantees,,1731751192181.697cd6a363e514bff0d4ec954fc4034b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/697cd6a363e514bff0d4ec954fc4034b/C/9b994a5406404b3fa89fea7191ed65e7 to hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/archive/data/default/TestAcidGuarantees/697cd6a363e514bff0d4ec954fc4034b/C/9b994a5406404b3fa89fea7191ed65e7 2024-11-16T10:00:16,982 DEBUG [StoreCloser-TestAcidGuarantees,,1731751192181.697cd6a363e514bff0d4ec954fc4034b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/697cd6a363e514bff0d4ec954fc4034b/C/e4b523607a6e49e8a22c3232d2fa54ec to hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/archive/data/default/TestAcidGuarantees/697cd6a363e514bff0d4ec954fc4034b/C/e4b523607a6e49e8a22c3232d2fa54ec 2024-11-16T10:00:16,982 DEBUG [StoreCloser-TestAcidGuarantees,,1731751192181.697cd6a363e514bff0d4ec954fc4034b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/697cd6a363e514bff0d4ec954fc4034b/C/1eda3944ab0448908f50204d73526559 to hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/archive/data/default/TestAcidGuarantees/697cd6a363e514bff0d4ec954fc4034b/C/1eda3944ab0448908f50204d73526559 2024-11-16T10:00:16,983 DEBUG [StoreCloser-TestAcidGuarantees,,1731751192181.697cd6a363e514bff0d4ec954fc4034b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/697cd6a363e514bff0d4ec954fc4034b/C/fd7057e35b004c38881704f7e925903e to hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/archive/data/default/TestAcidGuarantees/697cd6a363e514bff0d4ec954fc4034b/C/fd7057e35b004c38881704f7e925903e 2024-11-16T10:00:16,984 DEBUG [StoreCloser-TestAcidGuarantees,,1731751192181.697cd6a363e514bff0d4ec954fc4034b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/697cd6a363e514bff0d4ec954fc4034b/C/db2e5ea5d8c048eaa89b7e96a9c2e23a to hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/archive/data/default/TestAcidGuarantees/697cd6a363e514bff0d4ec954fc4034b/C/db2e5ea5d8c048eaa89b7e96a9c2e23a 2024-11-16T10:00:16,984 DEBUG [StoreCloser-TestAcidGuarantees,,1731751192181.697cd6a363e514bff0d4ec954fc4034b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/697cd6a363e514bff0d4ec954fc4034b/C/cb29188c3e284c5aafb352c461876405 to hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/archive/data/default/TestAcidGuarantees/697cd6a363e514bff0d4ec954fc4034b/C/cb29188c3e284c5aafb352c461876405 2024-11-16T10:00:16,985 DEBUG [StoreCloser-TestAcidGuarantees,,1731751192181.697cd6a363e514bff0d4ec954fc4034b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/697cd6a363e514bff0d4ec954fc4034b/C/4c12439cbf7a45158cddfb8f850956bf to hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/archive/data/default/TestAcidGuarantees/697cd6a363e514bff0d4ec954fc4034b/C/4c12439cbf7a45158cddfb8f850956bf 2024-11-16T10:00:16,986 DEBUG [StoreCloser-TestAcidGuarantees,,1731751192181.697cd6a363e514bff0d4ec954fc4034b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/697cd6a363e514bff0d4ec954fc4034b/C/1a1514cb66b8456bab35a7ce294b1663 to hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/archive/data/default/TestAcidGuarantees/697cd6a363e514bff0d4ec954fc4034b/C/1a1514cb66b8456bab35a7ce294b1663 2024-11-16T10:00:16,987 DEBUG [StoreCloser-TestAcidGuarantees,,1731751192181.697cd6a363e514bff0d4ec954fc4034b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/697cd6a363e514bff0d4ec954fc4034b/C/699da832fda74291b53ee546204927b7 to hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/archive/data/default/TestAcidGuarantees/697cd6a363e514bff0d4ec954fc4034b/C/699da832fda74291b53ee546204927b7 2024-11-16T10:00:16,987 DEBUG [StoreCloser-TestAcidGuarantees,,1731751192181.697cd6a363e514bff0d4ec954fc4034b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/697cd6a363e514bff0d4ec954fc4034b/C/8afbac04ec764dc4b9dbddce6d7afb77 to hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/archive/data/default/TestAcidGuarantees/697cd6a363e514bff0d4ec954fc4034b/C/8afbac04ec764dc4b9dbddce6d7afb77 2024-11-16T10:00:16,988 DEBUG [StoreCloser-TestAcidGuarantees,,1731751192181.697cd6a363e514bff0d4ec954fc4034b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/697cd6a363e514bff0d4ec954fc4034b/C/22cb10ec5fd249f3ba682ac831965d6d to hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/archive/data/default/TestAcidGuarantees/697cd6a363e514bff0d4ec954fc4034b/C/22cb10ec5fd249f3ba682ac831965d6d 2024-11-16T10:00:16,989 DEBUG [StoreCloser-TestAcidGuarantees,,1731751192181.697cd6a363e514bff0d4ec954fc4034b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/697cd6a363e514bff0d4ec954fc4034b/C/475bfdd4b2ed4ab9b2d86b58ceb856f3 to hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/archive/data/default/TestAcidGuarantees/697cd6a363e514bff0d4ec954fc4034b/C/475bfdd4b2ed4ab9b2d86b58ceb856f3 2024-11-16T10:00:16,989 DEBUG [StoreCloser-TestAcidGuarantees,,1731751192181.697cd6a363e514bff0d4ec954fc4034b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/697cd6a363e514bff0d4ec954fc4034b/C/6846a39ce2564b2cbeec112bc41ff608 to hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/archive/data/default/TestAcidGuarantees/697cd6a363e514bff0d4ec954fc4034b/C/6846a39ce2564b2cbeec112bc41ff608 2024-11-16T10:00:16,990 DEBUG [StoreCloser-TestAcidGuarantees,,1731751192181.697cd6a363e514bff0d4ec954fc4034b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/697cd6a363e514bff0d4ec954fc4034b/C/1d64ef6ef6ef4156812f479b5da065f6 to hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/archive/data/default/TestAcidGuarantees/697cd6a363e514bff0d4ec954fc4034b/C/1d64ef6ef6ef4156812f479b5da065f6 2024-11-16T10:00:16,991 DEBUG [StoreCloser-TestAcidGuarantees,,1731751192181.697cd6a363e514bff0d4ec954fc4034b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/697cd6a363e514bff0d4ec954fc4034b/C/9237610090d64e16b429d0a108aa1d65 to hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/archive/data/default/TestAcidGuarantees/697cd6a363e514bff0d4ec954fc4034b/C/9237610090d64e16b429d0a108aa1d65 2024-11-16T10:00:16,991 DEBUG [StoreCloser-TestAcidGuarantees,,1731751192181.697cd6a363e514bff0d4ec954fc4034b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/697cd6a363e514bff0d4ec954fc4034b/C/0c90ee06996748789e1d25093a7f1e25 to hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/archive/data/default/TestAcidGuarantees/697cd6a363e514bff0d4ec954fc4034b/C/0c90ee06996748789e1d25093a7f1e25 2024-11-16T10:00:16,992 DEBUG [StoreCloser-TestAcidGuarantees,,1731751192181.697cd6a363e514bff0d4ec954fc4034b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/697cd6a363e514bff0d4ec954fc4034b/C/0a8b4857d6014068ba12683461bf5532 to hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/archive/data/default/TestAcidGuarantees/697cd6a363e514bff0d4ec954fc4034b/C/0a8b4857d6014068ba12683461bf5532 2024-11-16T10:00:16,993 DEBUG [StoreCloser-TestAcidGuarantees,,1731751192181.697cd6a363e514bff0d4ec954fc4034b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/697cd6a363e514bff0d4ec954fc4034b/C/e4d5f1ee2bca43e5adfbd16e2b3aa12a to hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/archive/data/default/TestAcidGuarantees/697cd6a363e514bff0d4ec954fc4034b/C/e4d5f1ee2bca43e5adfbd16e2b3aa12a 2024-11-16T10:00:16,994 DEBUG [StoreCloser-TestAcidGuarantees,,1731751192181.697cd6a363e514bff0d4ec954fc4034b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/697cd6a363e514bff0d4ec954fc4034b/C/2b55b0ed5f8942d1a3f6a2eb0a32fc72 to hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/archive/data/default/TestAcidGuarantees/697cd6a363e514bff0d4ec954fc4034b/C/2b55b0ed5f8942d1a3f6a2eb0a32fc72 2024-11-16T10:00:16,997 DEBUG [RS_CLOSE_REGION-regionserver/037c58a19ddb:0-0 {event_type=M_RS_CLOSE_REGION, pid=140}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/697cd6a363e514bff0d4ec954fc4034b/recovered.edits/445.seqid, newMaxSeqId=445, maxSeqId=1 2024-11-16T10:00:16,998 INFO [RS_CLOSE_REGION-regionserver/037c58a19ddb:0-0 {event_type=M_RS_CLOSE_REGION, pid=140}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1731751192181.697cd6a363e514bff0d4ec954fc4034b. 2024-11-16T10:00:16,998 DEBUG [RS_CLOSE_REGION-regionserver/037c58a19ddb:0-0 {event_type=M_RS_CLOSE_REGION, pid=140}] regionserver.HRegion(1676): Region close journal for 697cd6a363e514bff0d4ec954fc4034b: Waiting for close lock at 1731751215206Running coprocessor pre-close hooks at 1731751215206Disabling compacts and flushes for region at 1731751215206Disabling writes for close at 1731751215701 (+495 ms)Obtaining lock to block concurrent updates at 1731751215701Preparing flush snapshotting stores in 697cd6a363e514bff0d4ec954fc4034b at 1731751215701Finished memstore snapshotting TestAcidGuarantees,,1731751192181.697cd6a363e514bff0d4ec954fc4034b., syncing WAL and waiting on mvcc, flushsize=dataSize=27480, getHeapSize=72720, getOffHeapSize=0, getCellsCount=600 at 1731751215702 (+1 ms)Flushing stores of TestAcidGuarantees,,1731751192181.697cd6a363e514bff0d4ec954fc4034b. at 1731751215702Flushing 697cd6a363e514bff0d4ec954fc4034b/A: creating writer at 1731751215702Flushing 697cd6a363e514bff0d4ec954fc4034b/A: appending metadata at 1731751215704 (+2 ms)Flushing 697cd6a363e514bff0d4ec954fc4034b/A: closing flushed file at 1731751215704Flushing 697cd6a363e514bff0d4ec954fc4034b/B: creating writer at 1731751216111 (+407 ms)Flushing 697cd6a363e514bff0d4ec954fc4034b/B: appending metadata at 1731751216112 (+1 ms)Flushing 697cd6a363e514bff0d4ec954fc4034b/B: closing flushed file at 1731751216112Flushing 697cd6a363e514bff0d4ec954fc4034b/C: creating writer at 1731751216519 (+407 ms)Flushing 697cd6a363e514bff0d4ec954fc4034b/C: appending metadata at 1731751216521 (+2 ms)Flushing 697cd6a363e514bff0d4ec954fc4034b/C: closing flushed file at 1731751216521Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@6d8063bc: reopening flushed file at 1731751216928 (+407 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@45d1beea: reopening flushed file at 1731751216931 (+3 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@16f63692: reopening flushed file at 1731751216934 (+3 ms)Finished flush of dataSize ~26.84 KB/27480, heapSize ~71.02 KB/72720, currentSize=0 B/0 for 697cd6a363e514bff0d4ec954fc4034b in 1237ms, sequenceid=442, compaction requested=true at 1731751216938 (+4 ms)Writing region close event to WAL at 1731751216995 (+57 ms)Running coprocessor post-close hooks at 1731751216998 (+3 ms)Closed at 1731751216998 2024-11-16T10:00:17,000 INFO [RS_CLOSE_REGION-regionserver/037c58a19ddb:0-0 {event_type=M_RS_CLOSE_REGION, pid=140}] handler.UnassignRegionHandler(157): Closed 697cd6a363e514bff0d4ec954fc4034b 2024-11-16T10:00:17,000 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=139 updating hbase:meta row=697cd6a363e514bff0d4ec954fc4034b, regionState=CLOSED 2024-11-16T10:00:17,001 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=140, ppid=139, state=RUNNABLE, hasLock=false; CloseRegionProcedure 697cd6a363e514bff0d4ec954fc4034b, server=037c58a19ddb,37267,1731751077844 because future has completed 2024-11-16T10:00:17,003 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=140, resume processing ppid=139 2024-11-16T10:00:17,003 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=140, ppid=139, state=SUCCESS, hasLock=false; CloseRegionProcedure 697cd6a363e514bff0d4ec954fc4034b, server=037c58a19ddb,37267,1731751077844 in 1.9480 sec 2024-11-16T10:00:17,005 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=139, resume processing ppid=138 2024-11-16T10:00:17,005 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=139, ppid=138, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=697cd6a363e514bff0d4ec954fc4034b, UNASSIGN in 1.9520 sec 2024-11-16T10:00:17,006 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=138, resume processing ppid=137 2024-11-16T10:00:17,007 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=138, ppid=137, state=SUCCESS, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees in 1.9550 sec 2024-11-16T10:00:17,008 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731751217008"}]},"ts":"1731751217008"} 2024-11-16T10:00:17,009 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLED in hbase:meta 2024-11-16T10:00:17,009 INFO [PEWorker-4 {}] procedure.DisableTableProcedure(296): Set TestAcidGuarantees to state=DISABLED 2024-11-16T10:00:17,010 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=137, state=SUCCESS, hasLock=false; DisableTableProcedure table=TestAcidGuarantees in 1.9630 sec 2024-11-16T10:00:17,193 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37877 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=137 2024-11-16T10:00:17,193 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DISABLE, Table Name: default:TestAcidGuarantees completed 2024-11-16T10:00:17,194 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37877 {}] master.HMaster$5(2570): Client=jenkins//172.17.0.2 delete TestAcidGuarantees 2024-11-16T10:00:17,195 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37877 {}] procedure2.ProcedureExecutor(1139): Stored pid=141, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees 2024-11-16T10:00:17,195 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(103): Waiting for RIT for pid=141, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-16T10:00:17,196 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37877 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=141 2024-11-16T10:00:17,196 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(115): Deleting regions from filesystem for pid=141, state=RUNNABLE:DELETE_TABLE_CLEAR_FS_LAYOUT, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-16T10:00:17,198 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(131): ARCHIVING hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/697cd6a363e514bff0d4ec954fc4034b 2024-11-16T10:00:17,200 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(159): Archiving [FileablePath, hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/697cd6a363e514bff0d4ec954fc4034b/A, FileablePath, hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/697cd6a363e514bff0d4ec954fc4034b/B, FileablePath, hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/697cd6a363e514bff0d4ec954fc4034b/C, FileablePath, hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/697cd6a363e514bff0d4ec954fc4034b/recovered.edits] 2024-11-16T10:00:17,202 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/697cd6a363e514bff0d4ec954fc4034b/A/3ffdd34e5ac64da0aad053a05f694142 to hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/archive/data/default/TestAcidGuarantees/697cd6a363e514bff0d4ec954fc4034b/A/3ffdd34e5ac64da0aad053a05f694142 2024-11-16T10:00:17,203 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/697cd6a363e514bff0d4ec954fc4034b/A/9c2b874e4fd24e10ab7b089c59b0aeb0 to hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/archive/data/default/TestAcidGuarantees/697cd6a363e514bff0d4ec954fc4034b/A/9c2b874e4fd24e10ab7b089c59b0aeb0 2024-11-16T10:00:17,203 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/697cd6a363e514bff0d4ec954fc4034b/A/9fe9bcf9a3c74206a24135894d172f1c to hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/archive/data/default/TestAcidGuarantees/697cd6a363e514bff0d4ec954fc4034b/A/9fe9bcf9a3c74206a24135894d172f1c 2024-11-16T10:00:17,204 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/697cd6a363e514bff0d4ec954fc4034b/A/feebd758ab0e40b086db26d4446b7716 to hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/archive/data/default/TestAcidGuarantees/697cd6a363e514bff0d4ec954fc4034b/A/feebd758ab0e40b086db26d4446b7716 2024-11-16T10:00:17,206 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/697cd6a363e514bff0d4ec954fc4034b/B/07c17bac7c0644d5965d02c99e4723ef to hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/archive/data/default/TestAcidGuarantees/697cd6a363e514bff0d4ec954fc4034b/B/07c17bac7c0644d5965d02c99e4723ef 2024-11-16T10:00:17,207 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/697cd6a363e514bff0d4ec954fc4034b/B/7c6833f12db243bdb8260f018e1a1b8c to hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/archive/data/default/TestAcidGuarantees/697cd6a363e514bff0d4ec954fc4034b/B/7c6833f12db243bdb8260f018e1a1b8c 2024-11-16T10:00:17,208 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/697cd6a363e514bff0d4ec954fc4034b/B/8262e61da0be43eaacaeb6971c33d6ad to hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/archive/data/default/TestAcidGuarantees/697cd6a363e514bff0d4ec954fc4034b/B/8262e61da0be43eaacaeb6971c33d6ad 2024-11-16T10:00:17,208 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/697cd6a363e514bff0d4ec954fc4034b/B/a7655708fdb94d7fb618714a81f63a33 to hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/archive/data/default/TestAcidGuarantees/697cd6a363e514bff0d4ec954fc4034b/B/a7655708fdb94d7fb618714a81f63a33 2024-11-16T10:00:17,210 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/697cd6a363e514bff0d4ec954fc4034b/C/344c6db7c2164b52a9e08f25363cf264 to hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/archive/data/default/TestAcidGuarantees/697cd6a363e514bff0d4ec954fc4034b/C/344c6db7c2164b52a9e08f25363cf264 2024-11-16T10:00:17,211 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/697cd6a363e514bff0d4ec954fc4034b/C/8b495ff68e964d87919267942138b50c to hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/archive/data/default/TestAcidGuarantees/697cd6a363e514bff0d4ec954fc4034b/C/8b495ff68e964d87919267942138b50c 2024-11-16T10:00:17,212 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/697cd6a363e514bff0d4ec954fc4034b/C/add9632fbd004800979d682fb30d05a0 to hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/archive/data/default/TestAcidGuarantees/697cd6a363e514bff0d4ec954fc4034b/C/add9632fbd004800979d682fb30d05a0 2024-11-16T10:00:17,212 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/697cd6a363e514bff0d4ec954fc4034b/C/e579817550f14a259b993196a4a5c56d to hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/archive/data/default/TestAcidGuarantees/697cd6a363e514bff0d4ec954fc4034b/C/e579817550f14a259b993196a4a5c56d 2024-11-16T10:00:17,214 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/697cd6a363e514bff0d4ec954fc4034b/recovered.edits/445.seqid to hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/archive/data/default/TestAcidGuarantees/697cd6a363e514bff0d4ec954fc4034b/recovered.edits/445.seqid 2024-11-16T10:00:17,215 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(610): Deleted hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/697cd6a363e514bff0d4ec954fc4034b 2024-11-16T10:00:17,215 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(313): Archived TestAcidGuarantees regions 2024-11-16T10:00:17,216 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(120): Deleting regions from META for pid=141, state=RUNNABLE:DELETE_TABLE_REMOVE_FROM_META, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-16T10:00:17,218 WARN [PEWorker-3 {}] procedure.DeleteTableProcedure(370): Deleting some vestigial 1 rows of TestAcidGuarantees from hbase:meta 2024-11-16T10:00:17,220 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(407): Removing 'TestAcidGuarantees' descriptor. 2024-11-16T10:00:17,221 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(126): Deleting assignment state for pid=141, state=RUNNABLE:DELETE_TABLE_UNASSIGN_REGIONS, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-16T10:00:17,221 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(397): Removing 'TestAcidGuarantees' from region states. 2024-11-16T10:00:17,221 DEBUG [PEWorker-3 {}] assignment.RegionStateStore(723): Delete {"totalColumns":1,"row":"TestAcidGuarantees,,1731751192181.697cd6a363e514bff0d4ec954fc4034b.","families":{"info":[{"qualifier":"","vlen":0,"tag":[],"timestamp":"1731751217221"}]},"ts":"9223372036854775807"} 2024-11-16T10:00:17,222 INFO [PEWorker-3 {}] assignment.RegionStateStore(562): Deleted 1 regions from META 2024-11-16T10:00:17,222 DEBUG [PEWorker-3 {}] assignment.RegionStateStore(563): Deleted regions: [{ENCODED => 697cd6a363e514bff0d4ec954fc4034b, NAME => 'TestAcidGuarantees,,1731751192181.697cd6a363e514bff0d4ec954fc4034b.', STARTKEY => '', ENDKEY => ''}] 2024-11-16T10:00:17,222 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(401): Marking 'TestAcidGuarantees' as deleted. 2024-11-16T10:00:17,223 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Delete {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":0,"tag":[],"timestamp":"1731751217222"}]},"ts":"9223372036854775807"} 2024-11-16T10:00:17,224 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(867): Deleted table TestAcidGuarantees state from META 2024-11-16T10:00:17,224 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(133): Finished pid=141, state=RUNNABLE:DELETE_TABLE_POST_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-16T10:00:17,225 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=141, state=SUCCESS, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees in 30 msec 2024-11-16T10:00:17,303 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37877 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=141 2024-11-16T10:00:17,303 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocator(219): Clear meta cache for TestAcidGuarantees 2024-11-16T10:00:17,303 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DELETE, Table Name: default:TestAcidGuarantees completed 2024-11-16T10:00:17,312 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: TestAcidGuaranteesWithNoInMemCompaction#testScanAtomicity Thread=241 (was 242), OpenFileDescriptor=451 (was 459), MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=375 (was 304) - SystemLoadAverage LEAK? -, ProcessCount=11 (was 11), AvailableMemoryMB=5819 (was 5849) 2024-11-16T10:00:17,320 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: TestAcidGuaranteesWithNoInMemCompaction#testMobGetAtomicity Thread=241, OpenFileDescriptor=451, MaxFileDescriptor=1048576, SystemLoadAverage=375, ProcessCount=11, AvailableMemoryMB=5819 2024-11-16T10:00:17,321 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37877 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (131072) is too small, which might cause very frequent flushing. 2024-11-16T10:00:17,322 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37877 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.2 create 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'NONE'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-16T10:00:17,322 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37877 {}] procedure2.ProcedureExecutor(1139): Stored pid=142, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=TestAcidGuarantees 2024-11-16T10:00:17,323 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=142, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_PRE_OPERATION 2024-11-16T10:00:17,323 DEBUG [PEWorker-5 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:17,323 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37877 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.2 procedure request for creating table: namespace: "default" qualifier: "TestAcidGuarantees" procId is: 142 2024-11-16T10:00:17,324 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37877 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=142 2024-11-16T10:00:17,324 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=142, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-11-16T10:00:17,328 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37591 is added to blk_1073742364_1540 (size=959) 2024-11-16T10:00:17,432 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37877 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=142 2024-11-16T10:00:17,539 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(153): Removing adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_TestAcidGuarantees 2024-11-16T10:00:17,643 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37877 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=142 2024-11-16T10:00:17,730 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => 75a5a39ee7bf008864556f27870615b9, NAME => 'TestAcidGuarantees,,1731751217321.75a5a39ee7bf008864556f27870615b9.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'NONE', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592 2024-11-16T10:00:17,734 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37591 is added to blk_1073742365_1541 (size=53) 2024-11-16T10:00:17,953 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37877 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=142 2024-11-16T10:00:18,135 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1731751217321.75a5a39ee7bf008864556f27870615b9.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-16T10:00:18,135 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1722): Closing 75a5a39ee7bf008864556f27870615b9, disabling compactions & flushes 2024-11-16T10:00:18,135 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1731751217321.75a5a39ee7bf008864556f27870615b9. 2024-11-16T10:00:18,135 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1731751217321.75a5a39ee7bf008864556f27870615b9. 2024-11-16T10:00:18,135 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1731751217321.75a5a39ee7bf008864556f27870615b9. after waiting 0 ms 2024-11-16T10:00:18,135 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1731751217321.75a5a39ee7bf008864556f27870615b9. 2024-11-16T10:00:18,135 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1731751217321.75a5a39ee7bf008864556f27870615b9. 2024-11-16T10:00:18,135 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1676): Region close journal for 75a5a39ee7bf008864556f27870615b9: Waiting for close lock at 1731751218135Disabling compacts and flushes for region at 1731751218135Disabling writes for close at 1731751218135Writing region close event to WAL at 1731751218135Closed at 1731751218135 2024-11-16T10:00:18,136 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=142, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ADD_TO_META 2024-11-16T10:00:18,136 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"TestAcidGuarantees,,1731751217321.75a5a39ee7bf008864556f27870615b9.","families":{"info":[{"qualifier":"regioninfo","vlen":52,"tag":[],"timestamp":"1731751218136"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1731751218136"}]},"ts":"1731751218136"} 2024-11-16T10:00:18,138 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-11-16T10:00:18,138 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=142, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-11-16T10:00:18,138 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731751218138"}]},"ts":"1731751218138"} 2024-11-16T10:00:18,139 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLING in hbase:meta 2024-11-16T10:00:18,140 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=143, ppid=142, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=75a5a39ee7bf008864556f27870615b9, ASSIGN}] 2024-11-16T10:00:18,140 INFO [PEWorker-2 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=143, ppid=142, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=75a5a39ee7bf008864556f27870615b9, ASSIGN 2024-11-16T10:00:18,141 INFO [PEWorker-2 {}] assignment.TransitRegionStateProcedure(269): Starting pid=143, ppid=142, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=75a5a39ee7bf008864556f27870615b9, ASSIGN; state=OFFLINE, location=037c58a19ddb,37267,1731751077844; forceNewPlan=false, retain=false 2024-11-16T10:00:18,292 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=143 updating hbase:meta row=75a5a39ee7bf008864556f27870615b9, regionState=OPENING, regionLocation=037c58a19ddb,37267,1731751077844 2024-11-16T10:00:18,293 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=143, ppid=142, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=75a5a39ee7bf008864556f27870615b9, ASSIGN because future has completed 2024-11-16T10:00:18,293 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=144, ppid=143, state=RUNNABLE, hasLock=false; OpenRegionProcedure 75a5a39ee7bf008864556f27870615b9, server=037c58a19ddb,37267,1731751077844}] 2024-11-16T10:00:18,447 INFO [RS_OPEN_REGION-regionserver/037c58a19ddb:0-0 {event_type=M_RS_OPEN_REGION, pid=144}] handler.AssignRegionHandler(132): Open TestAcidGuarantees,,1731751217321.75a5a39ee7bf008864556f27870615b9. 2024-11-16T10:00:18,447 DEBUG [RS_OPEN_REGION-regionserver/037c58a19ddb:0-0 {event_type=M_RS_OPEN_REGION, pid=144}] regionserver.HRegion(7752): Opening region: {ENCODED => 75a5a39ee7bf008864556f27870615b9, NAME => 'TestAcidGuarantees,,1731751217321.75a5a39ee7bf008864556f27870615b9.', STARTKEY => '', ENDKEY => ''} 2024-11-16T10:00:18,448 DEBUG [RS_OPEN_REGION-regionserver/037c58a19ddb:0-0 {event_type=M_RS_OPEN_REGION, pid=144}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestAcidGuarantees 75a5a39ee7bf008864556f27870615b9 2024-11-16T10:00:18,448 DEBUG [RS_OPEN_REGION-regionserver/037c58a19ddb:0-0 {event_type=M_RS_OPEN_REGION, pid=144}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1731751217321.75a5a39ee7bf008864556f27870615b9.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-16T10:00:18,448 DEBUG [RS_OPEN_REGION-regionserver/037c58a19ddb:0-0 {event_type=M_RS_OPEN_REGION, pid=144}] regionserver.HRegion(7794): checking encryption for 75a5a39ee7bf008864556f27870615b9 2024-11-16T10:00:18,448 DEBUG [RS_OPEN_REGION-regionserver/037c58a19ddb:0-0 {event_type=M_RS_OPEN_REGION, pid=144}] regionserver.HRegion(7797): checking classloading for 75a5a39ee7bf008864556f27870615b9 2024-11-16T10:00:18,449 INFO [StoreOpener-75a5a39ee7bf008864556f27870615b9-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family A of region 75a5a39ee7bf008864556f27870615b9 2024-11-16T10:00:18,450 INFO [StoreOpener-75a5a39ee7bf008864556f27870615b9-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 75a5a39ee7bf008864556f27870615b9 columnFamilyName A 2024-11-16T10:00:18,450 DEBUG [StoreOpener-75a5a39ee7bf008864556f27870615b9-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:18,450 INFO [StoreOpener-75a5a39ee7bf008864556f27870615b9-1 {}] regionserver.HStore(327): Store=75a5a39ee7bf008864556f27870615b9/A, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-16T10:00:18,450 INFO [StoreOpener-75a5a39ee7bf008864556f27870615b9-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family B of region 75a5a39ee7bf008864556f27870615b9 2024-11-16T10:00:18,451 INFO [StoreOpener-75a5a39ee7bf008864556f27870615b9-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 75a5a39ee7bf008864556f27870615b9 columnFamilyName B 2024-11-16T10:00:18,451 DEBUG [StoreOpener-75a5a39ee7bf008864556f27870615b9-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:18,452 INFO [StoreOpener-75a5a39ee7bf008864556f27870615b9-1 {}] regionserver.HStore(327): Store=75a5a39ee7bf008864556f27870615b9/B, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-16T10:00:18,452 INFO [StoreOpener-75a5a39ee7bf008864556f27870615b9-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family C of region 75a5a39ee7bf008864556f27870615b9 2024-11-16T10:00:18,452 INFO [StoreOpener-75a5a39ee7bf008864556f27870615b9-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 75a5a39ee7bf008864556f27870615b9 columnFamilyName C 2024-11-16T10:00:18,452 DEBUG [StoreOpener-75a5a39ee7bf008864556f27870615b9-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:18,453 INFO [StoreOpener-75a5a39ee7bf008864556f27870615b9-1 {}] regionserver.HStore(327): Store=75a5a39ee7bf008864556f27870615b9/C, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-16T10:00:18,453 DEBUG [RS_OPEN_REGION-regionserver/037c58a19ddb:0-0 {event_type=M_RS_OPEN_REGION, pid=144}] regionserver.HRegion(1038): replaying wal for 75a5a39ee7bf008864556f27870615b9 2024-11-16T10:00:18,453 DEBUG [RS_OPEN_REGION-regionserver/037c58a19ddb:0-0 {event_type=M_RS_OPEN_REGION, pid=144}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/75a5a39ee7bf008864556f27870615b9 2024-11-16T10:00:18,454 DEBUG [RS_OPEN_REGION-regionserver/037c58a19ddb:0-0 {event_type=M_RS_OPEN_REGION, pid=144}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/75a5a39ee7bf008864556f27870615b9 2024-11-16T10:00:18,454 DEBUG [RS_OPEN_REGION-regionserver/037c58a19ddb:0-0 {event_type=M_RS_OPEN_REGION, pid=144}] regionserver.HRegion(1048): stopping wal replay for 75a5a39ee7bf008864556f27870615b9 2024-11-16T10:00:18,454 DEBUG [RS_OPEN_REGION-regionserver/037c58a19ddb:0-0 {event_type=M_RS_OPEN_REGION, pid=144}] regionserver.HRegion(1060): Cleaning up temporary data for 75a5a39ee7bf008864556f27870615b9 2024-11-16T10:00:18,455 DEBUG [RS_OPEN_REGION-regionserver/037c58a19ddb:0-0 {event_type=M_RS_OPEN_REGION, pid=144}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table TestAcidGuarantees descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-11-16T10:00:18,455 DEBUG [RS_OPEN_REGION-regionserver/037c58a19ddb:0-0 {event_type=M_RS_OPEN_REGION, pid=144}] regionserver.HRegion(1093): writing seq id for 75a5a39ee7bf008864556f27870615b9 2024-11-16T10:00:18,457 DEBUG [RS_OPEN_REGION-regionserver/037c58a19ddb:0-0 {event_type=M_RS_OPEN_REGION, pid=144}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/75a5a39ee7bf008864556f27870615b9/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-16T10:00:18,457 INFO [RS_OPEN_REGION-regionserver/037c58a19ddb:0-0 {event_type=M_RS_OPEN_REGION, pid=144}] regionserver.HRegion(1114): Opened 75a5a39ee7bf008864556f27870615b9; next sequenceid=2; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=62163134, jitterRate=-0.07369711995124817}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-11-16T10:00:18,457 DEBUG [RS_OPEN_REGION-regionserver/037c58a19ddb:0-0 {event_type=M_RS_OPEN_REGION, pid=144}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 75a5a39ee7bf008864556f27870615b9 2024-11-16T10:00:18,458 DEBUG [RS_OPEN_REGION-regionserver/037c58a19ddb:0-0 {event_type=M_RS_OPEN_REGION, pid=144}] regionserver.HRegion(1006): Region open journal for 75a5a39ee7bf008864556f27870615b9: Running coprocessor pre-open hook at 1731751218448Writing region info on filesystem at 1731751218448Initializing all the Stores at 1731751218448Instantiating store for column family {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731751218448Instantiating store for column family {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731751218448Instantiating store for column family {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731751218449 (+1 ms)Cleaning up temporary data from old regions at 1731751218454 (+5 ms)Running coprocessor post-open hooks at 1731751218457 (+3 ms)Region opened successfully at 1731751218458 (+1 ms) 2024-11-16T10:00:18,459 INFO [RS_OPEN_REGION-regionserver/037c58a19ddb:0-0 {event_type=M_RS_OPEN_REGION, pid=144}] regionserver.HRegionServer(2236): Post open deploy tasks for TestAcidGuarantees,,1731751217321.75a5a39ee7bf008864556f27870615b9., pid=144, masterSystemTime=1731751218445 2024-11-16T10:00:18,460 DEBUG [RS_OPEN_REGION-regionserver/037c58a19ddb:0-0 {event_type=M_RS_OPEN_REGION, pid=144}] regionserver.HRegionServer(2266): Finished post open deploy task for TestAcidGuarantees,,1731751217321.75a5a39ee7bf008864556f27870615b9. 2024-11-16T10:00:18,460 INFO [RS_OPEN_REGION-regionserver/037c58a19ddb:0-0 {event_type=M_RS_OPEN_REGION, pid=144}] handler.AssignRegionHandler(153): Opened TestAcidGuarantees,,1731751217321.75a5a39ee7bf008864556f27870615b9. 2024-11-16T10:00:18,461 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=143 updating hbase:meta row=75a5a39ee7bf008864556f27870615b9, regionState=OPEN, openSeqNum=2, regionLocation=037c58a19ddb,37267,1731751077844 2024-11-16T10:00:18,462 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=144, ppid=143, state=RUNNABLE, hasLock=false; OpenRegionProcedure 75a5a39ee7bf008864556f27870615b9, server=037c58a19ddb,37267,1731751077844 because future has completed 2024-11-16T10:00:18,463 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37877 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=142 2024-11-16T10:00:18,464 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=144, resume processing ppid=143 2024-11-16T10:00:18,464 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=144, ppid=143, state=SUCCESS, hasLock=false; OpenRegionProcedure 75a5a39ee7bf008864556f27870615b9, server=037c58a19ddb,37267,1731751077844 in 169 msec 2024-11-16T10:00:18,466 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=143, resume processing ppid=142 2024-11-16T10:00:18,466 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=143, ppid=142, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=75a5a39ee7bf008864556f27870615b9, ASSIGN in 325 msec 2024-11-16T10:00:18,466 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=142, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-11-16T10:00:18,466 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731751218466"}]},"ts":"1731751218466"} 2024-11-16T10:00:18,467 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLED in hbase:meta 2024-11-16T10:00:18,468 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=142, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_POST_OPERATION 2024-11-16T10:00:18,469 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=142, state=SUCCESS, hasLock=false; CreateTableProcedure table=TestAcidGuarantees in 1.1460 sec 2024-11-16T10:00:19,473 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37877 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=142 2024-11-16T10:00:19,473 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: default:TestAcidGuarantees completed 2024-11-16T10:00:19,474 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5e76e6cf, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T10:00:19,474 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 037c58a19ddb,37877,-1 for getting cluster id 2024-11-16T10:00:19,474 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-16T10:00:19,475 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '1fe46ee8-cb44-44b1-8251-37103629f786' 2024-11-16T10:00:19,475 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-16T10:00:19,475 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "1fe46ee8-cb44-44b1-8251-37103629f786" 2024-11-16T10:00:19,476 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@23b7b638, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T10:00:19,476 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [037c58a19ddb,37877,-1] 2024-11-16T10:00:19,476 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-16T10:00:19,476 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T10:00:19,477 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:50234, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-16T10:00:19,477 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4125c327, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T10:00:19,478 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-16T10:00:19,478 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=037c58a19ddb,37267,1731751077844, seqNum=-1] 2024-11-16T10:00:19,479 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-16T10:00:19,479 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:49208, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-16T10:00:19,480 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-11-16T10:00:19,481 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncConnectionImpl(321): The fetched master address is 037c58a19ddb,37877,1731751077131 2024-11-16T10:00:19,481 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@30154d13 2024-11-16T10:00:19,481 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-11-16T10:00:19,482 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:50236, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-11-16T10:00:19,483 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37877 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (131072) is too small, which might cause very frequent flushing. 2024-11-16T10:00:19,483 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37877 {}] master.HMaster$14(2863): Client=jenkins//172.17.0.2 modify table TestAcidGuarantees from 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'NONE', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} to 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'NONE', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', MOB_THRESHOLD => '4', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', IS_MOB => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-16T10:00:19,484 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37877 {}] procedure2.ProcedureExecutor(1139): Stored pid=145, state=RUNNABLE:MODIFY_TABLE_PREPARE, hasLock=false; ModifyTableProcedure table=TestAcidGuarantees 2024-11-16T10:00:19,492 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37591 is added to blk_1073742366_1542 (size=995) 2024-11-16T10:00:19,893 DEBUG [PEWorker-3 {}] util.FSTableDescriptors(591): Deleted hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/.tabledesc/.tableinfo.0000000001.959 2024-11-16T10:00:19,893 INFO [PEWorker-3 {}] util.FSTableDescriptors(361): Updated tableinfo=hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/.tabledesc/.tableinfo.0000000002.995 2024-11-16T10:00:19,896 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=146, ppid=145, state=RUNNABLE:REOPEN_TABLE_REGIONS_GET_REGIONS, hasLock=false; ReopenTableRegionsProcedure table=TestAcidGuarantees}] 2024-11-16T10:00:19,897 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=147, ppid=146, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=75a5a39ee7bf008864556f27870615b9, REOPEN/MOVE}] 2024-11-16T10:00:19,898 INFO [PEWorker-2 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=147, ppid=146, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=75a5a39ee7bf008864556f27870615b9, REOPEN/MOVE 2024-11-16T10:00:19,899 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=147 updating hbase:meta row=75a5a39ee7bf008864556f27870615b9, regionState=CLOSING, regionLocation=037c58a19ddb,37267,1731751077844 2024-11-16T10:00:19,900 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=147, ppid=146, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=75a5a39ee7bf008864556f27870615b9, REOPEN/MOVE because future has completed 2024-11-16T10:00:19,900 DEBUG [PEWorker-1 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-11-16T10:00:19,900 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=148, ppid=147, state=RUNNABLE, hasLock=false; CloseRegionProcedure 75a5a39ee7bf008864556f27870615b9, server=037c58a19ddb,37267,1731751077844}] 2024-11-16T10:00:20,052 INFO [RS_CLOSE_REGION-regionserver/037c58a19ddb:0-0 {event_type=M_RS_CLOSE_REGION, pid=148}] handler.UnassignRegionHandler(122): Close 75a5a39ee7bf008864556f27870615b9 2024-11-16T10:00:20,052 DEBUG [RS_CLOSE_REGION-regionserver/037c58a19ddb:0-0 {event_type=M_RS_CLOSE_REGION, pid=148}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-11-16T10:00:20,052 DEBUG [RS_CLOSE_REGION-regionserver/037c58a19ddb:0-0 {event_type=M_RS_CLOSE_REGION, pid=148}] regionserver.HRegion(1722): Closing 75a5a39ee7bf008864556f27870615b9, disabling compactions & flushes 2024-11-16T10:00:20,052 INFO [RS_CLOSE_REGION-regionserver/037c58a19ddb:0-0 {event_type=M_RS_CLOSE_REGION, pid=148}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1731751217321.75a5a39ee7bf008864556f27870615b9. 2024-11-16T10:00:20,052 DEBUG [RS_CLOSE_REGION-regionserver/037c58a19ddb:0-0 {event_type=M_RS_CLOSE_REGION, pid=148}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1731751217321.75a5a39ee7bf008864556f27870615b9. 2024-11-16T10:00:20,052 DEBUG [RS_CLOSE_REGION-regionserver/037c58a19ddb:0-0 {event_type=M_RS_CLOSE_REGION, pid=148}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1731751217321.75a5a39ee7bf008864556f27870615b9. after waiting 0 ms 2024-11-16T10:00:20,052 DEBUG [RS_CLOSE_REGION-regionserver/037c58a19ddb:0-0 {event_type=M_RS_CLOSE_REGION, pid=148}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1731751217321.75a5a39ee7bf008864556f27870615b9. 2024-11-16T10:00:20,055 DEBUG [RS_CLOSE_REGION-regionserver/037c58a19ddb:0-0 {event_type=M_RS_CLOSE_REGION, pid=148}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/75a5a39ee7bf008864556f27870615b9/recovered.edits/4.seqid, newMaxSeqId=4, maxSeqId=1 2024-11-16T10:00:20,056 INFO [RS_CLOSE_REGION-regionserver/037c58a19ddb:0-0 {event_type=M_RS_CLOSE_REGION, pid=148}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1731751217321.75a5a39ee7bf008864556f27870615b9. 2024-11-16T10:00:20,056 DEBUG [RS_CLOSE_REGION-regionserver/037c58a19ddb:0-0 {event_type=M_RS_CLOSE_REGION, pid=148}] regionserver.HRegion(1676): Region close journal for 75a5a39ee7bf008864556f27870615b9: Waiting for close lock at 1731751220052Running coprocessor pre-close hooks at 1731751220052Disabling compacts and flushes for region at 1731751220052Disabling writes for close at 1731751220052Writing region close event to WAL at 1731751220053 (+1 ms)Running coprocessor post-close hooks at 1731751220056 (+3 ms)Closed at 1731751220056 2024-11-16T10:00:20,056 WARN [RS_CLOSE_REGION-regionserver/037c58a19ddb:0-0 {event_type=M_RS_CLOSE_REGION, pid=148}] regionserver.HRegionServer(3299): Not adding moved region record: 75a5a39ee7bf008864556f27870615b9 to self. 2024-11-16T10:00:20,057 INFO [RS_CLOSE_REGION-regionserver/037c58a19ddb:0-0 {event_type=M_RS_CLOSE_REGION, pid=148}] handler.UnassignRegionHandler(157): Closed 75a5a39ee7bf008864556f27870615b9 2024-11-16T10:00:20,058 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=147 updating hbase:meta row=75a5a39ee7bf008864556f27870615b9, regionState=CLOSED 2024-11-16T10:00:20,059 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=148, ppid=147, state=RUNNABLE, hasLock=false; CloseRegionProcedure 75a5a39ee7bf008864556f27870615b9, server=037c58a19ddb,37267,1731751077844 because future has completed 2024-11-16T10:00:20,061 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=148, resume processing ppid=147 2024-11-16T10:00:20,061 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=148, ppid=147, state=SUCCESS, hasLock=false; CloseRegionProcedure 75a5a39ee7bf008864556f27870615b9, server=037c58a19ddb,37267,1731751077844 in 159 msec 2024-11-16T10:00:20,061 INFO [PEWorker-2 {}] assignment.TransitRegionStateProcedure(269): Starting pid=147, ppid=146, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=75a5a39ee7bf008864556f27870615b9, REOPEN/MOVE; state=CLOSED, location=037c58a19ddb,37267,1731751077844; forceNewPlan=false, retain=true 2024-11-16T10:00:20,212 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=147 updating hbase:meta row=75a5a39ee7bf008864556f27870615b9, regionState=OPENING, regionLocation=037c58a19ddb,37267,1731751077844 2024-11-16T10:00:20,213 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=147, ppid=146, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=75a5a39ee7bf008864556f27870615b9, REOPEN/MOVE because future has completed 2024-11-16T10:00:20,214 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=149, ppid=147, state=RUNNABLE, hasLock=false; OpenRegionProcedure 75a5a39ee7bf008864556f27870615b9, server=037c58a19ddb,37267,1731751077844}] 2024-11-16T10:00:20,368 INFO [RS_OPEN_REGION-regionserver/037c58a19ddb:0-0 {event_type=M_RS_OPEN_REGION, pid=149}] handler.AssignRegionHandler(132): Open TestAcidGuarantees,,1731751217321.75a5a39ee7bf008864556f27870615b9. 2024-11-16T10:00:20,368 DEBUG [RS_OPEN_REGION-regionserver/037c58a19ddb:0-0 {event_type=M_RS_OPEN_REGION, pid=149}] regionserver.HRegion(7752): Opening region: {ENCODED => 75a5a39ee7bf008864556f27870615b9, NAME => 'TestAcidGuarantees,,1731751217321.75a5a39ee7bf008864556f27870615b9.', STARTKEY => '', ENDKEY => ''} 2024-11-16T10:00:20,368 DEBUG [RS_OPEN_REGION-regionserver/037c58a19ddb:0-0 {event_type=M_RS_OPEN_REGION, pid=149}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestAcidGuarantees 75a5a39ee7bf008864556f27870615b9 2024-11-16T10:00:20,368 DEBUG [RS_OPEN_REGION-regionserver/037c58a19ddb:0-0 {event_type=M_RS_OPEN_REGION, pid=149}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1731751217321.75a5a39ee7bf008864556f27870615b9.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-16T10:00:20,368 DEBUG [RS_OPEN_REGION-regionserver/037c58a19ddb:0-0 {event_type=M_RS_OPEN_REGION, pid=149}] regionserver.HRegion(7794): checking encryption for 75a5a39ee7bf008864556f27870615b9 2024-11-16T10:00:20,368 DEBUG [RS_OPEN_REGION-regionserver/037c58a19ddb:0-0 {event_type=M_RS_OPEN_REGION, pid=149}] regionserver.HRegion(7797): checking classloading for 75a5a39ee7bf008864556f27870615b9 2024-11-16T10:00:20,369 INFO [StoreOpener-75a5a39ee7bf008864556f27870615b9-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family A of region 75a5a39ee7bf008864556f27870615b9 2024-11-16T10:00:20,370 INFO [StoreOpener-75a5a39ee7bf008864556f27870615b9-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 75a5a39ee7bf008864556f27870615b9 columnFamilyName A 2024-11-16T10:00:20,371 DEBUG [StoreOpener-75a5a39ee7bf008864556f27870615b9-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:20,371 INFO [StoreOpener-75a5a39ee7bf008864556f27870615b9-1 {}] regionserver.HStore(327): Store=75a5a39ee7bf008864556f27870615b9/A, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-16T10:00:20,371 INFO [StoreOpener-75a5a39ee7bf008864556f27870615b9-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family B of region 75a5a39ee7bf008864556f27870615b9 2024-11-16T10:00:20,372 INFO [StoreOpener-75a5a39ee7bf008864556f27870615b9-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 75a5a39ee7bf008864556f27870615b9 columnFamilyName B 2024-11-16T10:00:20,372 DEBUG [StoreOpener-75a5a39ee7bf008864556f27870615b9-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:20,372 INFO [StoreOpener-75a5a39ee7bf008864556f27870615b9-1 {}] regionserver.HStore(327): Store=75a5a39ee7bf008864556f27870615b9/B, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-16T10:00:20,372 INFO [StoreOpener-75a5a39ee7bf008864556f27870615b9-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family C of region 75a5a39ee7bf008864556f27870615b9 2024-11-16T10:00:20,373 INFO [StoreOpener-75a5a39ee7bf008864556f27870615b9-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 75a5a39ee7bf008864556f27870615b9 columnFamilyName C 2024-11-16T10:00:20,373 DEBUG [StoreOpener-75a5a39ee7bf008864556f27870615b9-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:20,373 INFO [StoreOpener-75a5a39ee7bf008864556f27870615b9-1 {}] regionserver.HStore(327): Store=75a5a39ee7bf008864556f27870615b9/C, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-16T10:00:20,373 DEBUG [RS_OPEN_REGION-regionserver/037c58a19ddb:0-0 {event_type=M_RS_OPEN_REGION, pid=149}] regionserver.HRegion(1038): replaying wal for 75a5a39ee7bf008864556f27870615b9 2024-11-16T10:00:20,374 DEBUG [RS_OPEN_REGION-regionserver/037c58a19ddb:0-0 {event_type=M_RS_OPEN_REGION, pid=149}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/75a5a39ee7bf008864556f27870615b9 2024-11-16T10:00:20,375 DEBUG [RS_OPEN_REGION-regionserver/037c58a19ddb:0-0 {event_type=M_RS_OPEN_REGION, pid=149}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/75a5a39ee7bf008864556f27870615b9 2024-11-16T10:00:20,375 DEBUG [RS_OPEN_REGION-regionserver/037c58a19ddb:0-0 {event_type=M_RS_OPEN_REGION, pid=149}] regionserver.HRegion(1048): stopping wal replay for 75a5a39ee7bf008864556f27870615b9 2024-11-16T10:00:20,375 DEBUG [RS_OPEN_REGION-regionserver/037c58a19ddb:0-0 {event_type=M_RS_OPEN_REGION, pid=149}] regionserver.HRegion(1060): Cleaning up temporary data for 75a5a39ee7bf008864556f27870615b9 2024-11-16T10:00:20,375 DEBUG [RS_OPEN_REGION-regionserver/037c58a19ddb:0-0 {event_type=M_RS_OPEN_REGION, pid=149}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table TestAcidGuarantees descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-11-16T10:00:20,376 DEBUG [RS_OPEN_REGION-regionserver/037c58a19ddb:0-0 {event_type=M_RS_OPEN_REGION, pid=149}] regionserver.HRegion(1093): writing seq id for 75a5a39ee7bf008864556f27870615b9 2024-11-16T10:00:20,377 INFO [RS_OPEN_REGION-regionserver/037c58a19ddb:0-0 {event_type=M_RS_OPEN_REGION, pid=149}] regionserver.HRegion(1114): Opened 75a5a39ee7bf008864556f27870615b9; next sequenceid=5; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=72916038, jitterRate=0.08653363585472107}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-11-16T10:00:20,377 DEBUG [RS_OPEN_REGION-regionserver/037c58a19ddb:0-0 {event_type=M_RS_OPEN_REGION, pid=149}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 75a5a39ee7bf008864556f27870615b9 2024-11-16T10:00:20,377 DEBUG [RS_OPEN_REGION-regionserver/037c58a19ddb:0-0 {event_type=M_RS_OPEN_REGION, pid=149}] regionserver.HRegion(1006): Region open journal for 75a5a39ee7bf008864556f27870615b9: Running coprocessor pre-open hook at 1731751220368Writing region info on filesystem at 1731751220368Initializing all the Stores at 1731751220369 (+1 ms)Instantiating store for column family {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', MOB_THRESHOLD => '4', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', IS_MOB => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731751220369Instantiating store for column family {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731751220369Instantiating store for column family {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731751220369Cleaning up temporary data from old regions at 1731751220375 (+6 ms)Running coprocessor post-open hooks at 1731751220377 (+2 ms)Region opened successfully at 1731751220377 2024-11-16T10:00:20,378 INFO [RS_OPEN_REGION-regionserver/037c58a19ddb:0-0 {event_type=M_RS_OPEN_REGION, pid=149}] regionserver.HRegionServer(2236): Post open deploy tasks for TestAcidGuarantees,,1731751217321.75a5a39ee7bf008864556f27870615b9., pid=149, masterSystemTime=1731751220365 2024-11-16T10:00:20,379 DEBUG [RS_OPEN_REGION-regionserver/037c58a19ddb:0-0 {event_type=M_RS_OPEN_REGION, pid=149}] regionserver.HRegionServer(2266): Finished post open deploy task for TestAcidGuarantees,,1731751217321.75a5a39ee7bf008864556f27870615b9. 2024-11-16T10:00:20,380 INFO [RS_OPEN_REGION-regionserver/037c58a19ddb:0-0 {event_type=M_RS_OPEN_REGION, pid=149}] handler.AssignRegionHandler(153): Opened TestAcidGuarantees,,1731751217321.75a5a39ee7bf008864556f27870615b9. 2024-11-16T10:00:20,380 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=147 updating hbase:meta row=75a5a39ee7bf008864556f27870615b9, regionState=OPEN, openSeqNum=5, regionLocation=037c58a19ddb,37267,1731751077844 2024-11-16T10:00:20,381 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=149, ppid=147, state=RUNNABLE, hasLock=false; OpenRegionProcedure 75a5a39ee7bf008864556f27870615b9, server=037c58a19ddb,37267,1731751077844 because future has completed 2024-11-16T10:00:20,383 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=149, resume processing ppid=147 2024-11-16T10:00:20,383 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=149, ppid=147, state=SUCCESS, hasLock=false; OpenRegionProcedure 75a5a39ee7bf008864556f27870615b9, server=037c58a19ddb,37267,1731751077844 in 168 msec 2024-11-16T10:00:20,384 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=147, resume processing ppid=146 2024-11-16T10:00:20,384 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=147, ppid=146, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=75a5a39ee7bf008864556f27870615b9, REOPEN/MOVE in 486 msec 2024-11-16T10:00:20,386 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=146, resume processing ppid=145 2024-11-16T10:00:20,386 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=146, ppid=145, state=SUCCESS, hasLock=false; ReopenTableRegionsProcedure table=TestAcidGuarantees in 490 msec 2024-11-16T10:00:20,388 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=145, state=SUCCESS, hasLock=false; ModifyTableProcedure table=TestAcidGuarantees in 904 msec 2024-11-16T10:00:20,389 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37877 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=145 2024-11-16T10:00:20,389 INFO [RPCClient-NioEventLoopGroup-4-12 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: MODIFY_COLUMN_FAMILY, Table Name: default:TestAcidGuarantees completed 2024-11-16T10:00:20,390 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3e2aa61c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T10:00:20,390 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 037c58a19ddb,37877,-1 for getting cluster id 2024-11-16T10:00:20,390 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-16T10:00:20,391 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '1fe46ee8-cb44-44b1-8251-37103629f786' 2024-11-16T10:00:20,391 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-16T10:00:20,391 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "1fe46ee8-cb44-44b1-8251-37103629f786" 2024-11-16T10:00:20,391 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@34293636, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T10:00:20,391 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [037c58a19ddb,37877,-1] 2024-11-16T10:00:20,391 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-16T10:00:20,391 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T10:00:20,392 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:59374, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-16T10:00:20,392 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4b2b6e0c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T10:00:20,393 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@40503a36, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T10:00:20,393 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 037c58a19ddb,37877,-1 for getting cluster id 2024-11-16T10:00:20,393 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-16T10:00:20,394 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '1fe46ee8-cb44-44b1-8251-37103629f786' 2024-11-16T10:00:20,394 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-16T10:00:20,394 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "1fe46ee8-cb44-44b1-8251-37103629f786" 2024-11-16T10:00:20,394 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@fb8ae86, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T10:00:20,394 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [037c58a19ddb,37877,-1] 2024-11-16T10:00:20,394 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-16T10:00:20,394 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T10:00:20,395 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:59388, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-16T10:00:20,395 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@27235ce, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T10:00:20,396 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@52f72e1a, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T10:00:20,396 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 037c58a19ddb,37877,-1 for getting cluster id 2024-11-16T10:00:20,396 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-16T10:00:20,396 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '1fe46ee8-cb44-44b1-8251-37103629f786' 2024-11-16T10:00:20,397 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-16T10:00:20,397 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "1fe46ee8-cb44-44b1-8251-37103629f786" 2024-11-16T10:00:20,397 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@416f1746, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T10:00:20,397 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [037c58a19ddb,37877,-1] 2024-11-16T10:00:20,397 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-16T10:00:20,397 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T10:00:20,398 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:59410, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-16T10:00:20,398 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@23852bd3, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T10:00:20,398 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@47173f60, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T10:00:20,399 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 037c58a19ddb,37877,-1 for getting cluster id 2024-11-16T10:00:20,399 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-16T10:00:20,399 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '1fe46ee8-cb44-44b1-8251-37103629f786' 2024-11-16T10:00:20,399 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-16T10:00:20,399 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "1fe46ee8-cb44-44b1-8251-37103629f786" 2024-11-16T10:00:20,399 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@53e6d877, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T10:00:20,399 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [037c58a19ddb,37877,-1] 2024-11-16T10:00:20,399 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-16T10:00:20,400 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T10:00:20,400 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:59430, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-16T10:00:20,401 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7183c201, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T10:00:20,401 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@13700528, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T10:00:20,401 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 037c58a19ddb,37877,-1 for getting cluster id 2024-11-16T10:00:20,401 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-16T10:00:20,402 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '1fe46ee8-cb44-44b1-8251-37103629f786' 2024-11-16T10:00:20,402 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-16T10:00:20,402 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "1fe46ee8-cb44-44b1-8251-37103629f786" 2024-11-16T10:00:20,402 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3801243b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T10:00:20,402 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [037c58a19ddb,37877,-1] 2024-11-16T10:00:20,402 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-16T10:00:20,402 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T10:00:20,403 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:59446, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-16T10:00:20,403 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@606bb93a, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T10:00:20,404 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3666b173, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T10:00:20,404 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 037c58a19ddb,37877,-1 for getting cluster id 2024-11-16T10:00:20,404 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-16T10:00:20,404 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '1fe46ee8-cb44-44b1-8251-37103629f786' 2024-11-16T10:00:20,404 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-16T10:00:20,404 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "1fe46ee8-cb44-44b1-8251-37103629f786" 2024-11-16T10:00:20,404 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@14b68b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T10:00:20,405 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [037c58a19ddb,37877,-1] 2024-11-16T10:00:20,405 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-16T10:00:20,405 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T10:00:20,405 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:59452, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-16T10:00:20,406 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@63a966bc, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T10:00:20,406 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@77ff027c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T10:00:20,406 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 037c58a19ddb,37877,-1 for getting cluster id 2024-11-16T10:00:20,406 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-16T10:00:20,407 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '1fe46ee8-cb44-44b1-8251-37103629f786' 2024-11-16T10:00:20,407 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-16T10:00:20,407 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "1fe46ee8-cb44-44b1-8251-37103629f786" 2024-11-16T10:00:20,407 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@20d840dd, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T10:00:20,407 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [037c58a19ddb,37877,-1] 2024-11-16T10:00:20,408 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-16T10:00:20,408 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T10:00:20,408 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:59478, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-16T10:00:20,409 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@59199719, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T10:00:20,409 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2637386d, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T10:00:20,409 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 037c58a19ddb,37877,-1 for getting cluster id 2024-11-16T10:00:20,409 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-16T10:00:20,410 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '1fe46ee8-cb44-44b1-8251-37103629f786' 2024-11-16T10:00:20,410 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-16T10:00:20,410 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "1fe46ee8-cb44-44b1-8251-37103629f786" 2024-11-16T10:00:20,410 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@563f0995, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T10:00:20,410 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [037c58a19ddb,37877,-1] 2024-11-16T10:00:20,410 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-16T10:00:20,411 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T10:00:20,411 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:59500, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-16T10:00:20,411 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@36690996, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T10:00:20,412 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2455dde6, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T10:00:20,412 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 037c58a19ddb,37877,-1 for getting cluster id 2024-11-16T10:00:20,412 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-16T10:00:20,413 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '1fe46ee8-cb44-44b1-8251-37103629f786' 2024-11-16T10:00:20,413 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-16T10:00:20,413 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "1fe46ee8-cb44-44b1-8251-37103629f786" 2024-11-16T10:00:20,413 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@16824238, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T10:00:20,413 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [037c58a19ddb,37877,-1] 2024-11-16T10:00:20,413 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-16T10:00:20,413 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T10:00:20,414 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:59528, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-16T10:00:20,414 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@208874ea, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T10:00:20,414 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6cd1a81c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T10:00:20,414 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 037c58a19ddb,37877,-1 for getting cluster id 2024-11-16T10:00:20,415 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-16T10:00:20,415 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '1fe46ee8-cb44-44b1-8251-37103629f786' 2024-11-16T10:00:20,415 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-16T10:00:20,415 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "1fe46ee8-cb44-44b1-8251-37103629f786" 2024-11-16T10:00:20,415 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7472a72b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T10:00:20,415 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [037c58a19ddb,37877,-1] 2024-11-16T10:00:20,416 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-16T10:00:20,416 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T10:00:20,416 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:59532, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-16T10:00:20,417 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2035b2ac, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T10:00:20,417 DEBUG [Thread-2429 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-16T10:00:20,417 DEBUG [Thread-2427 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-16T10:00:20,417 DEBUG [Thread-2433 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-16T10:00:20,417 DEBUG [Thread-2416 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-16T10:00:20,417 DEBUG [Thread-2420 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-16T10:00:20,418 DEBUG [Thread-2431 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-16T10:00:20,418 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=037c58a19ddb,37267,1731751077844, seqNum=-1] 2024-11-16T10:00:20,418 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=037c58a19ddb,37267,1731751077844, seqNum=-1] 2024-11-16T10:00:20,418 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=037c58a19ddb,37267,1731751077844, seqNum=-1] 2024-11-16T10:00:20,418 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=037c58a19ddb,37267,1731751077844, seqNum=-1] 2024-11-16T10:00:20,418 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-16T10:00:20,418 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-16T10:00:20,418 DEBUG [Thread-2418 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-16T10:00:20,418 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-16T10:00:20,419 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-16T10:00:20,419 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=037c58a19ddb,37267,1731751077844, seqNum=-1] 2024-11-16T10:00:20,419 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=037c58a19ddb,37267,1731751077844, seqNum=-1] 2024-11-16T10:00:20,419 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-16T10:00:20,419 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-16T10:00:20,419 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37877 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-16T10:00:20,419 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:34372, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-16T10:00:20,419 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:34374, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-16T10:00:20,420 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:34390, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-16T10:00:20,420 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:34398, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-16T10:00:20,420 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=037c58a19ddb,37267,1731751077844, seqNum=-1] 2024-11-16T10:00:20,420 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:34392, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-16T10:00:20,420 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-16T10:00:20,420 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37877 {}] procedure2.ProcedureExecutor(1139): Stored pid=150, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=150, table=TestAcidGuarantees 2024-11-16T10:00:20,420 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:34412, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-16T10:00:20,421 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1731751217321.75a5a39ee7bf008864556f27870615b9., hostname=037c58a19ddb,37267,1731751077844, seqNum=5] 2024-11-16T10:00:20,421 DEBUG [Thread-2422 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-16T10:00:20,421 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1731751217321.75a5a39ee7bf008864556f27870615b9., hostname=037c58a19ddb,37267,1731751077844, seqNum=5] 2024-11-16T10:00:20,421 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=150, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=150, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-16T10:00:20,422 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37877 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=150 2024-11-16T10:00:20,422 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=037c58a19ddb,37267,1731751077844, seqNum=-1] 2024-11-16T10:00:20,422 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-16T10:00:20,422 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1731751217321.75a5a39ee7bf008864556f27870615b9., hostname=037c58a19ddb,37267,1731751077844, seqNum=5] 2024-11-16T10:00:20,423 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1731751217321.75a5a39ee7bf008864556f27870615b9., hostname=037c58a19ddb,37267,1731751077844, seqNum=5] 2024-11-16T10:00:20,423 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1731751217321.75a5a39ee7bf008864556f27870615b9., hostname=037c58a19ddb,37267,1731751077844, seqNum=5] 2024-11-16T10:00:20,423 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=150, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=150, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-16T10:00:20,423 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=151, ppid=150, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-16T10:00:20,424 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1731751217321.75a5a39ee7bf008864556f27870615b9., hostname=037c58a19ddb,37267,1731751077844, seqNum=5] 2024-11-16T10:00:20,424 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:34428, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-16T10:00:20,424 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:34430, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-16T10:00:20,425 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1731751217321.75a5a39ee7bf008864556f27870615b9., hostname=037c58a19ddb,37267,1731751077844, seqNum=5] 2024-11-16T10:00:20,426 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1731751217321.75a5a39ee7bf008864556f27870615b9., hostname=037c58a19ddb,37267,1731751077844, seqNum=5] 2024-11-16T10:00:20,433 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] regionserver.HRegion(8855): Flush requested on 75a5a39ee7bf008864556f27870615b9 2024-11-16T10:00:20,433 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-16T10:00:20,433 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 75a5a39ee7bf008864556f27870615b9 3/3 column families, dataSize=53.67 KB heapSize=141.38 KB 2024-11-16T10:00:20,439 DEBUG [Thread-2425 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-16T10:00:20,440 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=037c58a19ddb,37267,1731751077844, seqNum=-1] 2024-11-16T10:00:20,440 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-16T10:00:20,441 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:34446, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-16T10:00:20,443 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1731751217321.75a5a39ee7bf008864556f27870615b9., hostname=037c58a19ddb,37267,1731751077844, seqNum=5] 2024-11-16T10:00:20,444 DEBUG [Thread-2414 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-16T10:00:20,445 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=037c58a19ddb,37267,1731751077844, seqNum=-1] 2024-11-16T10:00:20,445 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-16T10:00:20,447 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:34454, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-16T10:00:20,448 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1731751217321.75a5a39ee7bf008864556f27870615b9., hostname=037c58a19ddb,37267,1731751077844, seqNum=5] 2024-11-16T10:00:20,464 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111659cbafcb5f8d4b6eb3889e2cda08ed2d_75a5a39ee7bf008864556f27870615b9 is 50, key is test_row_0/A:col10/1731751220432/Put/seqid=0 2024-11-16T10:00:20,468 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=75a5a39ee7bf008864556f27870615b9, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T10:00:20,468 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34430 deadline: 1731751280463, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=75a5a39ee7bf008864556f27870615b9, server=037c58a19ddb,37267,1731751077844 2024-11-16T10:00:20,468 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731751217321.75a5a39ee7bf008864556f27870615b9., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 , the old value is region=TestAcidGuarantees,,1731751217321.75a5a39ee7bf008864556f27870615b9., hostname=037c58a19ddb,37267,1731751077844, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=75a5a39ee7bf008864556f27870615b9, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:00:20,468 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731751217321.75a5a39ee7bf008864556f27870615b9., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=75a5a39ee7bf008864556f27870615b9, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:00:20,468 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731751217321.75a5a39ee7bf008864556f27870615b9., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 because the exception is null or not the one we care about 2024-11-16T10:00:20,471 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=75a5a39ee7bf008864556f27870615b9, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T10:00:20,471 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] ipc.CallRunner(138): callId: 3 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34454 deadline: 1731751280465, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=75a5a39ee7bf008864556f27870615b9, server=037c58a19ddb,37267,1731751077844 2024-11-16T10:00:20,471 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=75a5a39ee7bf008864556f27870615b9, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T10:00:20,471 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=75a5a39ee7bf008864556f27870615b9, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T10:00:20,471 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731751217321.75a5a39ee7bf008864556f27870615b9., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 , the old value is region=TestAcidGuarantees,,1731751217321.75a5a39ee7bf008864556f27870615b9., hostname=037c58a19ddb,37267,1731751077844, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=75a5a39ee7bf008864556f27870615b9, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:00:20,471 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731751217321.75a5a39ee7bf008864556f27870615b9., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=75a5a39ee7bf008864556f27870615b9, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:00:20,471 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34428 deadline: 1731751280465, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=75a5a39ee7bf008864556f27870615b9, server=037c58a19ddb,37267,1731751077844 2024-11-16T10:00:20,471 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34392 deadline: 1731751280466, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=75a5a39ee7bf008864556f27870615b9, server=037c58a19ddb,37267,1731751077844 2024-11-16T10:00:20,471 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731751217321.75a5a39ee7bf008864556f27870615b9., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 because the exception is null or not the one we care about 2024-11-16T10:00:20,472 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=75a5a39ee7bf008864556f27870615b9, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T10:00:20,472 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731751217321.75a5a39ee7bf008864556f27870615b9., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 , the old value is region=TestAcidGuarantees,,1731751217321.75a5a39ee7bf008864556f27870615b9., hostname=037c58a19ddb,37267,1731751077844, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=75a5a39ee7bf008864556f27870615b9, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:00:20,472 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731751217321.75a5a39ee7bf008864556f27870615b9., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=75a5a39ee7bf008864556f27870615b9, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:00:20,472 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34412 deadline: 1731751280468, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=75a5a39ee7bf008864556f27870615b9, server=037c58a19ddb,37267,1731751077844 2024-11-16T10:00:20,472 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731751217321.75a5a39ee7bf008864556f27870615b9., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 because the exception is null or not the one we care about 2024-11-16T10:00:20,472 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731751217321.75a5a39ee7bf008864556f27870615b9., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 , the old value is region=TestAcidGuarantees,,1731751217321.75a5a39ee7bf008864556f27870615b9., hostname=037c58a19ddb,37267,1731751077844, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=75a5a39ee7bf008864556f27870615b9, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:00:20,472 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731751217321.75a5a39ee7bf008864556f27870615b9., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=75a5a39ee7bf008864556f27870615b9, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:00:20,472 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731751217321.75a5a39ee7bf008864556f27870615b9., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 , the old value is region=TestAcidGuarantees,,1731751217321.75a5a39ee7bf008864556f27870615b9., hostname=037c58a19ddb,37267,1731751077844, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=75a5a39ee7bf008864556f27870615b9, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:00:20,472 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731751217321.75a5a39ee7bf008864556f27870615b9., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 because the exception is null or not the one we care about 2024-11-16T10:00:20,472 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731751217321.75a5a39ee7bf008864556f27870615b9., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=75a5a39ee7bf008864556f27870615b9, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:00:20,472 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731751217321.75a5a39ee7bf008864556f27870615b9., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 because the exception is null or not the one we care about 2024-11-16T10:00:20,481 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37591 is added to blk_1073742367_1543 (size=12154) 2024-11-16T10:00:20,533 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37877 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=150 2024-11-16T10:00:20,575 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=75a5a39ee7bf008864556f27870615b9, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T10:00:20,575 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34430 deadline: 1731751280573, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=75a5a39ee7bf008864556f27870615b9, server=037c58a19ddb,37267,1731751077844 2024-11-16T10:00:20,575 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731751217321.75a5a39ee7bf008864556f27870615b9., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 , the old value is region=TestAcidGuarantees,,1731751217321.75a5a39ee7bf008864556f27870615b9., hostname=037c58a19ddb,37267,1731751077844, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=75a5a39ee7bf008864556f27870615b9, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:00:20,575 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731751217321.75a5a39ee7bf008864556f27870615b9., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=75a5a39ee7bf008864556f27870615b9, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:00:20,575 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731751217321.75a5a39ee7bf008864556f27870615b9., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 because the exception is null or not the one we care about 2024-11-16T10:00:20,576 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=37267 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=151 2024-11-16T10:00:20,576 DEBUG [RS_FLUSH_OPERATIONS-regionserver/037c58a19ddb:0-2 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731751217321.75a5a39ee7bf008864556f27870615b9. 2024-11-16T10:00:20,576 DEBUG [RS_FLUSH_OPERATIONS-regionserver/037c58a19ddb:0-2 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731751217321.75a5a39ee7bf008864556f27870615b9. as already flushing 2024-11-16T10:00:20,576 DEBUG [RS_FLUSH_OPERATIONS-regionserver/037c58a19ddb:0-2 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731751217321.75a5a39ee7bf008864556f27870615b9. 2024-11-16T10:00:20,576 ERROR [RS_FLUSH_OPERATIONS-regionserver/037c58a19ddb:0-2 {event_type=RS_FLUSH_REGIONS, pid=151}] handler.RSProcedureHandler(58): pid=151 java.io.IOException: Unable to complete flush {ENCODED => 75a5a39ee7bf008864556f27870615b9, NAME => 'TestAcidGuarantees,,1731751217321.75a5a39ee7bf008864556f27870615b9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T10:00:20,576 DEBUG [RS_FLUSH_OPERATIONS-regionserver/037c58a19ddb:0-2 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=151 java.io.IOException: Unable to complete flush {ENCODED => 75a5a39ee7bf008864556f27870615b9, NAME => 'TestAcidGuarantees,,1731751217321.75a5a39ee7bf008864556f27870615b9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T10:00:20,577 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37877 {}] master.HMaster(4177): Remote procedure failed, pid=151 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 75a5a39ee7bf008864556f27870615b9, NAME => 'TestAcidGuarantees,,1731751217321.75a5a39ee7bf008864556f27870615b9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 75a5a39ee7bf008864556f27870615b9, NAME => 'TestAcidGuarantees,,1731751217321.75a5a39ee7bf008864556f27870615b9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T10:00:20,585 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=75a5a39ee7bf008864556f27870615b9, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T10:00:20,585 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] ipc.CallRunner(138): callId: 4 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34454 deadline: 1731751280582, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=75a5a39ee7bf008864556f27870615b9, server=037c58a19ddb,37267,1731751077844 2024-11-16T10:00:20,585 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=75a5a39ee7bf008864556f27870615b9, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T10:00:20,585 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731751217321.75a5a39ee7bf008864556f27870615b9., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 , the old value is region=TestAcidGuarantees,,1731751217321.75a5a39ee7bf008864556f27870615b9., hostname=037c58a19ddb,37267,1731751077844, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=75a5a39ee7bf008864556f27870615b9, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:00:20,586 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731751217321.75a5a39ee7bf008864556f27870615b9., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=75a5a39ee7bf008864556f27870615b9, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:00:20,586 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34412 deadline: 1731751280583, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=75a5a39ee7bf008864556f27870615b9, server=037c58a19ddb,37267,1731751077844 2024-11-16T10:00:20,586 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731751217321.75a5a39ee7bf008864556f27870615b9., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 because the exception is null or not the one we care about 2024-11-16T10:00:20,586 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=75a5a39ee7bf008864556f27870615b9, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T10:00:20,586 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34392 deadline: 1731751280583, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=75a5a39ee7bf008864556f27870615b9, server=037c58a19ddb,37267,1731751077844 2024-11-16T10:00:20,586 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731751217321.75a5a39ee7bf008864556f27870615b9., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 , the old value is region=TestAcidGuarantees,,1731751217321.75a5a39ee7bf008864556f27870615b9., hostname=037c58a19ddb,37267,1731751077844, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=75a5a39ee7bf008864556f27870615b9, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:00:20,586 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731751217321.75a5a39ee7bf008864556f27870615b9., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=75a5a39ee7bf008864556f27870615b9, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:00:20,586 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=75a5a39ee7bf008864556f27870615b9, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T10:00:20,586 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731751217321.75a5a39ee7bf008864556f27870615b9., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 because the exception is null or not the one we care about 2024-11-16T10:00:20,586 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34428 deadline: 1731751280583, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=75a5a39ee7bf008864556f27870615b9, server=037c58a19ddb,37267,1731751077844 2024-11-16T10:00:20,586 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731751217321.75a5a39ee7bf008864556f27870615b9., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 , the old value is region=TestAcidGuarantees,,1731751217321.75a5a39ee7bf008864556f27870615b9., hostname=037c58a19ddb,37267,1731751077844, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=75a5a39ee7bf008864556f27870615b9, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:00:20,586 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731751217321.75a5a39ee7bf008864556f27870615b9., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=75a5a39ee7bf008864556f27870615b9, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:00:20,586 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731751217321.75a5a39ee7bf008864556f27870615b9., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 because the exception is null or not the one we care about 2024-11-16T10:00:20,586 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731751217321.75a5a39ee7bf008864556f27870615b9., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 , the old value is region=TestAcidGuarantees,,1731751217321.75a5a39ee7bf008864556f27870615b9., hostname=037c58a19ddb,37267,1731751077844, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=75a5a39ee7bf008864556f27870615b9, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:00:20,586 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731751217321.75a5a39ee7bf008864556f27870615b9., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=75a5a39ee7bf008864556f27870615b9, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:00:20,586 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731751217321.75a5a39ee7bf008864556f27870615b9., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 because the exception is null or not the one we care about 2024-11-16T10:00:20,729 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=37267 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=151 2024-11-16T10:00:20,729 DEBUG [RS_FLUSH_OPERATIONS-regionserver/037c58a19ddb:0-0 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731751217321.75a5a39ee7bf008864556f27870615b9. 2024-11-16T10:00:20,729 DEBUG [RS_FLUSH_OPERATIONS-regionserver/037c58a19ddb:0-0 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731751217321.75a5a39ee7bf008864556f27870615b9. as already flushing 2024-11-16T10:00:20,729 DEBUG [RS_FLUSH_OPERATIONS-regionserver/037c58a19ddb:0-0 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731751217321.75a5a39ee7bf008864556f27870615b9. 2024-11-16T10:00:20,729 ERROR [RS_FLUSH_OPERATIONS-regionserver/037c58a19ddb:0-0 {event_type=RS_FLUSH_REGIONS, pid=151}] handler.RSProcedureHandler(58): pid=151 java.io.IOException: Unable to complete flush {ENCODED => 75a5a39ee7bf008864556f27870615b9, NAME => 'TestAcidGuarantees,,1731751217321.75a5a39ee7bf008864556f27870615b9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T10:00:20,730 DEBUG [RS_FLUSH_OPERATIONS-regionserver/037c58a19ddb:0-0 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=151 java.io.IOException: Unable to complete flush {ENCODED => 75a5a39ee7bf008864556f27870615b9, NAME => 'TestAcidGuarantees,,1731751217321.75a5a39ee7bf008864556f27870615b9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T10:00:20,730 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37877 {}] master.HMaster(4177): Remote procedure failed, pid=151 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 75a5a39ee7bf008864556f27870615b9, NAME => 'TestAcidGuarantees,,1731751217321.75a5a39ee7bf008864556f27870615b9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 75a5a39ee7bf008864556f27870615b9, NAME => 'TestAcidGuarantees,,1731751217321.75a5a39ee7bf008864556f27870615b9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T10:00:20,743 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37877 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=150 2024-11-16T10:00:20,784 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=75a5a39ee7bf008864556f27870615b9, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T10:00:20,785 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34430 deadline: 1731751280782, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=75a5a39ee7bf008864556f27870615b9, server=037c58a19ddb,37267,1731751077844 2024-11-16T10:00:20,785 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731751217321.75a5a39ee7bf008864556f27870615b9., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 , the old value is region=TestAcidGuarantees,,1731751217321.75a5a39ee7bf008864556f27870615b9., hostname=037c58a19ddb,37267,1731751077844, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=75a5a39ee7bf008864556f27870615b9, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:00:20,785 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731751217321.75a5a39ee7bf008864556f27870615b9., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=75a5a39ee7bf008864556f27870615b9, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:00:20,785 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731751217321.75a5a39ee7bf008864556f27870615b9., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 because the exception is null or not the one we care about 2024-11-16T10:00:20,794 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=75a5a39ee7bf008864556f27870615b9, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T10:00:20,794 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] ipc.CallRunner(138): callId: 5 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34454 deadline: 1731751280793, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=75a5a39ee7bf008864556f27870615b9, server=037c58a19ddb,37267,1731751077844 2024-11-16T10:00:20,794 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731751217321.75a5a39ee7bf008864556f27870615b9., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 , the old value is region=TestAcidGuarantees,,1731751217321.75a5a39ee7bf008864556f27870615b9., hostname=037c58a19ddb,37267,1731751077844, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=75a5a39ee7bf008864556f27870615b9, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:00:20,794 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=75a5a39ee7bf008864556f27870615b9, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T10:00:20,794 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731751217321.75a5a39ee7bf008864556f27870615b9., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=75a5a39ee7bf008864556f27870615b9, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:00:20,794 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731751217321.75a5a39ee7bf008864556f27870615b9., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 because the exception is null or not the one we care about 2024-11-16T10:00:20,795 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34412 deadline: 1731751280793, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=75a5a39ee7bf008864556f27870615b9, server=037c58a19ddb,37267,1731751077844 2024-11-16T10:00:20,795 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=75a5a39ee7bf008864556f27870615b9, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T10:00:20,795 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731751217321.75a5a39ee7bf008864556f27870615b9., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 , the old value is region=TestAcidGuarantees,,1731751217321.75a5a39ee7bf008864556f27870615b9., hostname=037c58a19ddb,37267,1731751077844, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=75a5a39ee7bf008864556f27870615b9, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:00:20,795 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34392 deadline: 1731751280793, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=75a5a39ee7bf008864556f27870615b9, server=037c58a19ddb,37267,1731751077844 2024-11-16T10:00:20,795 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731751217321.75a5a39ee7bf008864556f27870615b9., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=75a5a39ee7bf008864556f27870615b9, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:00:20,795 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731751217321.75a5a39ee7bf008864556f27870615b9., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 because the exception is null or not the one we care about 2024-11-16T10:00:20,795 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731751217321.75a5a39ee7bf008864556f27870615b9., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 , the old value is region=TestAcidGuarantees,,1731751217321.75a5a39ee7bf008864556f27870615b9., hostname=037c58a19ddb,37267,1731751077844, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=75a5a39ee7bf008864556f27870615b9, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:00:20,795 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731751217321.75a5a39ee7bf008864556f27870615b9., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=75a5a39ee7bf008864556f27870615b9, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:00:20,795 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731751217321.75a5a39ee7bf008864556f27870615b9., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 because the exception is null or not the one we care about 2024-11-16T10:00:20,795 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=75a5a39ee7bf008864556f27870615b9, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T10:00:20,795 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34428 deadline: 1731751280793, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=75a5a39ee7bf008864556f27870615b9, server=037c58a19ddb,37267,1731751077844 2024-11-16T10:00:20,796 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731751217321.75a5a39ee7bf008864556f27870615b9., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 , the old value is region=TestAcidGuarantees,,1731751217321.75a5a39ee7bf008864556f27870615b9., hostname=037c58a19ddb,37267,1731751077844, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=75a5a39ee7bf008864556f27870615b9, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:00:20,796 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731751217321.75a5a39ee7bf008864556f27870615b9., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=75a5a39ee7bf008864556f27870615b9, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:00:20,796 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731751217321.75a5a39ee7bf008864556f27870615b9., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 because the exception is null or not the one we care about 2024-11-16T10:00:20,882 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:20,882 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=37267 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=151 2024-11-16T10:00:20,882 DEBUG [RS_FLUSH_OPERATIONS-regionserver/037c58a19ddb:0-1 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731751217321.75a5a39ee7bf008864556f27870615b9. 2024-11-16T10:00:20,883 DEBUG [RS_FLUSH_OPERATIONS-regionserver/037c58a19ddb:0-1 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731751217321.75a5a39ee7bf008864556f27870615b9. as already flushing 2024-11-16T10:00:20,883 DEBUG [RS_FLUSH_OPERATIONS-regionserver/037c58a19ddb:0-1 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731751217321.75a5a39ee7bf008864556f27870615b9. 2024-11-16T10:00:20,883 ERROR [RS_FLUSH_OPERATIONS-regionserver/037c58a19ddb:0-1 {event_type=RS_FLUSH_REGIONS, pid=151}] handler.RSProcedureHandler(58): pid=151 java.io.IOException: Unable to complete flush {ENCODED => 75a5a39ee7bf008864556f27870615b9, NAME => 'TestAcidGuarantees,,1731751217321.75a5a39ee7bf008864556f27870615b9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T10:00:20,883 DEBUG [RS_FLUSH_OPERATIONS-regionserver/037c58a19ddb:0-1 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=151 java.io.IOException: Unable to complete flush {ENCODED => 75a5a39ee7bf008864556f27870615b9, NAME => 'TestAcidGuarantees,,1731751217321.75a5a39ee7bf008864556f27870615b9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T10:00:20,883 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37877 {}] master.HMaster(4177): Remote procedure failed, pid=151 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 75a5a39ee7bf008864556f27870615b9, NAME => 'TestAcidGuarantees,,1731751217321.75a5a39ee7bf008864556f27870615b9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 75a5a39ee7bf008864556f27870615b9, NAME => 'TestAcidGuarantees,,1731751217321.75a5a39ee7bf008864556f27870615b9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T10:00:20,885 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111659cbafcb5f8d4b6eb3889e2cda08ed2d_75a5a39ee7bf008864556f27870615b9 to hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111659cbafcb5f8d4b6eb3889e2cda08ed2d_75a5a39ee7bf008864556f27870615b9 2024-11-16T10:00:20,886 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/75a5a39ee7bf008864556f27870615b9/.tmp/A/a08f489d2f324f8c94bc70f75b31e4c0, store: [table=TestAcidGuarantees family=A region=75a5a39ee7bf008864556f27870615b9] 2024-11-16T10:00:20,886 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/75a5a39ee7bf008864556f27870615b9/.tmp/A/a08f489d2f324f8c94bc70f75b31e4c0 is 175, key is test_row_0/A:col10/1731751220432/Put/seqid=0 2024-11-16T10:00:20,890 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37591 is added to blk_1073742368_1544 (size=30955) 2024-11-16T10:00:21,034 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=37267 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=151 2024-11-16T10:00:21,035 DEBUG [RS_FLUSH_OPERATIONS-regionserver/037c58a19ddb:0-2 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731751217321.75a5a39ee7bf008864556f27870615b9. 2024-11-16T10:00:21,035 DEBUG [RS_FLUSH_OPERATIONS-regionserver/037c58a19ddb:0-2 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731751217321.75a5a39ee7bf008864556f27870615b9. as already flushing 2024-11-16T10:00:21,035 DEBUG [RS_FLUSH_OPERATIONS-regionserver/037c58a19ddb:0-2 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731751217321.75a5a39ee7bf008864556f27870615b9. 2024-11-16T10:00:21,035 ERROR [RS_FLUSH_OPERATIONS-regionserver/037c58a19ddb:0-2 {event_type=RS_FLUSH_REGIONS, pid=151}] handler.RSProcedureHandler(58): pid=151 java.io.IOException: Unable to complete flush {ENCODED => 75a5a39ee7bf008864556f27870615b9, NAME => 'TestAcidGuarantees,,1731751217321.75a5a39ee7bf008864556f27870615b9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T10:00:21,035 DEBUG [RS_FLUSH_OPERATIONS-regionserver/037c58a19ddb:0-2 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=151 java.io.IOException: Unable to complete flush {ENCODED => 75a5a39ee7bf008864556f27870615b9, NAME => 'TestAcidGuarantees,,1731751217321.75a5a39ee7bf008864556f27870615b9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T10:00:21,036 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37877 {}] master.HMaster(4177): Remote procedure failed, pid=151 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 75a5a39ee7bf008864556f27870615b9, NAME => 'TestAcidGuarantees,,1731751217321.75a5a39ee7bf008864556f27870615b9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 75a5a39ee7bf008864556f27870615b9, NAME => 'TestAcidGuarantees,,1731751217321.75a5a39ee7bf008864556f27870615b9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T10:00:21,053 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37877 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=150 2024-11-16T10:00:21,094 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=75a5a39ee7bf008864556f27870615b9, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T10:00:21,094 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34430 deadline: 1731751281092, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=75a5a39ee7bf008864556f27870615b9, server=037c58a19ddb,37267,1731751077844 2024-11-16T10:00:21,095 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731751217321.75a5a39ee7bf008864556f27870615b9., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 , the old value is region=TestAcidGuarantees,,1731751217321.75a5a39ee7bf008864556f27870615b9., hostname=037c58a19ddb,37267,1731751077844, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=75a5a39ee7bf008864556f27870615b9, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:00:21,095 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731751217321.75a5a39ee7bf008864556f27870615b9., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=75a5a39ee7bf008864556f27870615b9, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:00:21,095 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731751217321.75a5a39ee7bf008864556f27870615b9., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 because the exception is null or not the one we care about 2024-11-16T10:00:21,104 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=75a5a39ee7bf008864556f27870615b9, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T10:00:21,104 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] ipc.CallRunner(138): callId: 6 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34454 deadline: 1731751281103, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=75a5a39ee7bf008864556f27870615b9, server=037c58a19ddb,37267,1731751077844 2024-11-16T10:00:21,104 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=75a5a39ee7bf008864556f27870615b9, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T10:00:21,104 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34412 deadline: 1731751281103, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=75a5a39ee7bf008864556f27870615b9, server=037c58a19ddb,37267,1731751077844 2024-11-16T10:00:21,105 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731751217321.75a5a39ee7bf008864556f27870615b9., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 , the old value is region=TestAcidGuarantees,,1731751217321.75a5a39ee7bf008864556f27870615b9., hostname=037c58a19ddb,37267,1731751077844, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=75a5a39ee7bf008864556f27870615b9, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:00:21,105 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731751217321.75a5a39ee7bf008864556f27870615b9., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=75a5a39ee7bf008864556f27870615b9, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:00:21,105 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731751217321.75a5a39ee7bf008864556f27870615b9., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 because the exception is null or not the one we care about 2024-11-16T10:00:21,105 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731751217321.75a5a39ee7bf008864556f27870615b9., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 , the old value is region=TestAcidGuarantees,,1731751217321.75a5a39ee7bf008864556f27870615b9., hostname=037c58a19ddb,37267,1731751077844, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=75a5a39ee7bf008864556f27870615b9, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:00:21,105 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=75a5a39ee7bf008864556f27870615b9, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T10:00:21,105 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731751217321.75a5a39ee7bf008864556f27870615b9., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=75a5a39ee7bf008864556f27870615b9, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:00:21,105 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731751217321.75a5a39ee7bf008864556f27870615b9., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 because the exception is null or not the one we care about 2024-11-16T10:00:21,105 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34392 deadline: 1731751281103, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=75a5a39ee7bf008864556f27870615b9, server=037c58a19ddb,37267,1731751077844 2024-11-16T10:00:21,105 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=75a5a39ee7bf008864556f27870615b9, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T10:00:21,105 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731751217321.75a5a39ee7bf008864556f27870615b9., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 , the old value is region=TestAcidGuarantees,,1731751217321.75a5a39ee7bf008864556f27870615b9., hostname=037c58a19ddb,37267,1731751077844, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=75a5a39ee7bf008864556f27870615b9, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:00:21,105 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34428 deadline: 1731751281103, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=75a5a39ee7bf008864556f27870615b9, server=037c58a19ddb,37267,1731751077844 2024-11-16T10:00:21,105 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731751217321.75a5a39ee7bf008864556f27870615b9., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=75a5a39ee7bf008864556f27870615b9, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:00:21,105 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731751217321.75a5a39ee7bf008864556f27870615b9., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 because the exception is null or not the one we care about 2024-11-16T10:00:21,105 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731751217321.75a5a39ee7bf008864556f27870615b9., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 , the old value is region=TestAcidGuarantees,,1731751217321.75a5a39ee7bf008864556f27870615b9., hostname=037c58a19ddb,37267,1731751077844, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=75a5a39ee7bf008864556f27870615b9, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:00:21,105 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731751217321.75a5a39ee7bf008864556f27870615b9., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=75a5a39ee7bf008864556f27870615b9, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:00:21,105 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731751217321.75a5a39ee7bf008864556f27870615b9., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 because the exception is null or not the one we care about 2024-11-16T10:00:21,187 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=37267 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=151 2024-11-16T10:00:21,188 DEBUG [RS_FLUSH_OPERATIONS-regionserver/037c58a19ddb:0-0 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731751217321.75a5a39ee7bf008864556f27870615b9. 2024-11-16T10:00:21,188 DEBUG [RS_FLUSH_OPERATIONS-regionserver/037c58a19ddb:0-0 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731751217321.75a5a39ee7bf008864556f27870615b9. as already flushing 2024-11-16T10:00:21,188 DEBUG [RS_FLUSH_OPERATIONS-regionserver/037c58a19ddb:0-0 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731751217321.75a5a39ee7bf008864556f27870615b9. 2024-11-16T10:00:21,188 ERROR [RS_FLUSH_OPERATIONS-regionserver/037c58a19ddb:0-0 {event_type=RS_FLUSH_REGIONS, pid=151}] handler.RSProcedureHandler(58): pid=151 java.io.IOException: Unable to complete flush {ENCODED => 75a5a39ee7bf008864556f27870615b9, NAME => 'TestAcidGuarantees,,1731751217321.75a5a39ee7bf008864556f27870615b9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T10:00:21,188 DEBUG [RS_FLUSH_OPERATIONS-regionserver/037c58a19ddb:0-0 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=151 java.io.IOException: Unable to complete flush {ENCODED => 75a5a39ee7bf008864556f27870615b9, NAME => 'TestAcidGuarantees,,1731751217321.75a5a39ee7bf008864556f27870615b9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T10:00:21,189 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37877 {}] master.HMaster(4177): Remote procedure failed, pid=151 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 75a5a39ee7bf008864556f27870615b9, NAME => 'TestAcidGuarantees,,1731751217321.75a5a39ee7bf008864556f27870615b9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 75a5a39ee7bf008864556f27870615b9, NAME => 'TestAcidGuarantees,,1731751217321.75a5a39ee7bf008864556f27870615b9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T10:00:21,290 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=16, memsize=20.1 K, hasBloomFilter=true, into tmp file hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/75a5a39ee7bf008864556f27870615b9/.tmp/A/a08f489d2f324f8c94bc70f75b31e4c0 2024-11-16T10:00:21,311 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/75a5a39ee7bf008864556f27870615b9/.tmp/B/932ed69b7379451ea7d13e6008653448 is 50, key is test_row_0/B:col10/1731751220432/Put/seqid=0 2024-11-16T10:00:21,315 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37591 is added to blk_1073742369_1545 (size=12001) 2024-11-16T10:00:21,340 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=37267 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=151 2024-11-16T10:00:21,341 DEBUG [RS_FLUSH_OPERATIONS-regionserver/037c58a19ddb:0-1 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731751217321.75a5a39ee7bf008864556f27870615b9. 2024-11-16T10:00:21,342 DEBUG [RS_FLUSH_OPERATIONS-regionserver/037c58a19ddb:0-1 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731751217321.75a5a39ee7bf008864556f27870615b9. as already flushing 2024-11-16T10:00:21,342 DEBUG [RS_FLUSH_OPERATIONS-regionserver/037c58a19ddb:0-1 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731751217321.75a5a39ee7bf008864556f27870615b9. 2024-11-16T10:00:21,342 ERROR [RS_FLUSH_OPERATIONS-regionserver/037c58a19ddb:0-1 {event_type=RS_FLUSH_REGIONS, pid=151}] handler.RSProcedureHandler(58): pid=151 java.io.IOException: Unable to complete flush {ENCODED => 75a5a39ee7bf008864556f27870615b9, NAME => 'TestAcidGuarantees,,1731751217321.75a5a39ee7bf008864556f27870615b9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T10:00:21,342 DEBUG [RS_FLUSH_OPERATIONS-regionserver/037c58a19ddb:0-1 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=151 java.io.IOException: Unable to complete flush {ENCODED => 75a5a39ee7bf008864556f27870615b9, NAME => 'TestAcidGuarantees,,1731751217321.75a5a39ee7bf008864556f27870615b9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T10:00:21,343 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37877 {}] master.HMaster(4177): Remote procedure failed, pid=151 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 75a5a39ee7bf008864556f27870615b9, NAME => 'TestAcidGuarantees,,1731751217321.75a5a39ee7bf008864556f27870615b9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 75a5a39ee7bf008864556f27870615b9, NAME => 'TestAcidGuarantees,,1731751217321.75a5a39ee7bf008864556f27870615b9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T10:00:21,494 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=37267 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=151 2024-11-16T10:00:21,494 DEBUG [RS_FLUSH_OPERATIONS-regionserver/037c58a19ddb:0-2 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731751217321.75a5a39ee7bf008864556f27870615b9. 2024-11-16T10:00:21,495 DEBUG [RS_FLUSH_OPERATIONS-regionserver/037c58a19ddb:0-2 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731751217321.75a5a39ee7bf008864556f27870615b9. as already flushing 2024-11-16T10:00:21,495 DEBUG [RS_FLUSH_OPERATIONS-regionserver/037c58a19ddb:0-2 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731751217321.75a5a39ee7bf008864556f27870615b9. 2024-11-16T10:00:21,495 ERROR [RS_FLUSH_OPERATIONS-regionserver/037c58a19ddb:0-2 {event_type=RS_FLUSH_REGIONS, pid=151}] handler.RSProcedureHandler(58): pid=151 java.io.IOException: Unable to complete flush {ENCODED => 75a5a39ee7bf008864556f27870615b9, NAME => 'TestAcidGuarantees,,1731751217321.75a5a39ee7bf008864556f27870615b9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T10:00:21,495 DEBUG [RS_FLUSH_OPERATIONS-regionserver/037c58a19ddb:0-2 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=151 java.io.IOException: Unable to complete flush {ENCODED => 75a5a39ee7bf008864556f27870615b9, NAME => 'TestAcidGuarantees,,1731751217321.75a5a39ee7bf008864556f27870615b9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T10:00:21,495 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37877 {}] master.HMaster(4177): Remote procedure failed, pid=151 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 75a5a39ee7bf008864556f27870615b9, NAME => 'TestAcidGuarantees,,1731751217321.75a5a39ee7bf008864556f27870615b9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 75a5a39ee7bf008864556f27870615b9, NAME => 'TestAcidGuarantees,,1731751217321.75a5a39ee7bf008864556f27870615b9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T10:00:21,563 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37877 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=150 2024-11-16T10:00:21,604 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=75a5a39ee7bf008864556f27870615b9, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T10:00:21,604 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34430 deadline: 1731751281603, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=75a5a39ee7bf008864556f27870615b9, server=037c58a19ddb,37267,1731751077844 2024-11-16T10:00:21,605 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731751217321.75a5a39ee7bf008864556f27870615b9., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 , the old value is region=TestAcidGuarantees,,1731751217321.75a5a39ee7bf008864556f27870615b9., hostname=037c58a19ddb,37267,1731751077844, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=75a5a39ee7bf008864556f27870615b9, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:00:21,605 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731751217321.75a5a39ee7bf008864556f27870615b9., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=75a5a39ee7bf008864556f27870615b9, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:00:21,605 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731751217321.75a5a39ee7bf008864556f27870615b9., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 because the exception is null or not the one we care about 2024-11-16T10:00:21,615 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=75a5a39ee7bf008864556f27870615b9, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T10:00:21,615 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34454 deadline: 1731751281613, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=75a5a39ee7bf008864556f27870615b9, server=037c58a19ddb,37267,1731751077844 2024-11-16T10:00:21,615 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=75a5a39ee7bf008864556f27870615b9, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T10:00:21,615 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731751217321.75a5a39ee7bf008864556f27870615b9., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 , the old value is region=TestAcidGuarantees,,1731751217321.75a5a39ee7bf008864556f27870615b9., hostname=037c58a19ddb,37267,1731751077844, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=75a5a39ee7bf008864556f27870615b9, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:00:21,615 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34412 deadline: 1731751281613, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=75a5a39ee7bf008864556f27870615b9, server=037c58a19ddb,37267,1731751077844 2024-11-16T10:00:21,615 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731751217321.75a5a39ee7bf008864556f27870615b9., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=75a5a39ee7bf008864556f27870615b9, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:00:21,615 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731751217321.75a5a39ee7bf008864556f27870615b9., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 because the exception is null or not the one we care about 2024-11-16T10:00:21,615 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731751217321.75a5a39ee7bf008864556f27870615b9., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 , the old value is region=TestAcidGuarantees,,1731751217321.75a5a39ee7bf008864556f27870615b9., hostname=037c58a19ddb,37267,1731751077844, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=75a5a39ee7bf008864556f27870615b9, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:00:21,615 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=75a5a39ee7bf008864556f27870615b9, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T10:00:21,616 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731751217321.75a5a39ee7bf008864556f27870615b9., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=75a5a39ee7bf008864556f27870615b9, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:00:21,616 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731751217321.75a5a39ee7bf008864556f27870615b9., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 because the exception is null or not the one we care about 2024-11-16T10:00:21,616 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34392 deadline: 1731751281613, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=75a5a39ee7bf008864556f27870615b9, server=037c58a19ddb,37267,1731751077844 2024-11-16T10:00:21,616 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731751217321.75a5a39ee7bf008864556f27870615b9., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 , the old value is region=TestAcidGuarantees,,1731751217321.75a5a39ee7bf008864556f27870615b9., hostname=037c58a19ddb,37267,1731751077844, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=75a5a39ee7bf008864556f27870615b9, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:00:21,616 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731751217321.75a5a39ee7bf008864556f27870615b9., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=75a5a39ee7bf008864556f27870615b9, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:00:21,616 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731751217321.75a5a39ee7bf008864556f27870615b9., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 because the exception is null or not the one we care about 2024-11-16T10:00:21,616 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=75a5a39ee7bf008864556f27870615b9, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T10:00:21,616 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34428 deadline: 1731751281613, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=75a5a39ee7bf008864556f27870615b9, server=037c58a19ddb,37267,1731751077844 2024-11-16T10:00:21,616 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731751217321.75a5a39ee7bf008864556f27870615b9., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 , the old value is region=TestAcidGuarantees,,1731751217321.75a5a39ee7bf008864556f27870615b9., hostname=037c58a19ddb,37267,1731751077844, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=75a5a39ee7bf008864556f27870615b9, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:00:21,616 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731751217321.75a5a39ee7bf008864556f27870615b9., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=75a5a39ee7bf008864556f27870615b9, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:00:21,616 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731751217321.75a5a39ee7bf008864556f27870615b9., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 because the exception is null or not the one we care about 2024-11-16T10:00:21,646 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=37267 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=151 2024-11-16T10:00:21,646 DEBUG [RS_FLUSH_OPERATIONS-regionserver/037c58a19ddb:0-0 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731751217321.75a5a39ee7bf008864556f27870615b9. 2024-11-16T10:00:21,647 DEBUG [RS_FLUSH_OPERATIONS-regionserver/037c58a19ddb:0-0 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731751217321.75a5a39ee7bf008864556f27870615b9. as already flushing 2024-11-16T10:00:21,647 DEBUG [RS_FLUSH_OPERATIONS-regionserver/037c58a19ddb:0-0 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731751217321.75a5a39ee7bf008864556f27870615b9. 2024-11-16T10:00:21,647 ERROR [RS_FLUSH_OPERATIONS-regionserver/037c58a19ddb:0-0 {event_type=RS_FLUSH_REGIONS, pid=151}] handler.RSProcedureHandler(58): pid=151 java.io.IOException: Unable to complete flush {ENCODED => 75a5a39ee7bf008864556f27870615b9, NAME => 'TestAcidGuarantees,,1731751217321.75a5a39ee7bf008864556f27870615b9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T10:00:21,647 DEBUG [RS_FLUSH_OPERATIONS-regionserver/037c58a19ddb:0-0 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=151 java.io.IOException: Unable to complete flush {ENCODED => 75a5a39ee7bf008864556f27870615b9, NAME => 'TestAcidGuarantees,,1731751217321.75a5a39ee7bf008864556f27870615b9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T10:00:21,648 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37877 {}] master.HMaster(4177): Remote procedure failed, pid=151 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 75a5a39ee7bf008864556f27870615b9, NAME => 'TestAcidGuarantees,,1731751217321.75a5a39ee7bf008864556f27870615b9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 75a5a39ee7bf008864556f27870615b9, NAME => 'TestAcidGuarantees,,1731751217321.75a5a39ee7bf008864556f27870615b9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T10:00:21,716 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=16 (bloomFilter=true), to=hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/75a5a39ee7bf008864556f27870615b9/.tmp/B/932ed69b7379451ea7d13e6008653448 2024-11-16T10:00:21,739 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/75a5a39ee7bf008864556f27870615b9/.tmp/C/ba3e707513614fbe871b02ce8b163935 is 50, key is test_row_0/C:col10/1731751220432/Put/seqid=0 2024-11-16T10:00:21,742 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37591 is added to blk_1073742370_1546 (size=12001) 2024-11-16T10:00:21,800 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=37267 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=151 2024-11-16T10:00:21,800 DEBUG [RS_FLUSH_OPERATIONS-regionserver/037c58a19ddb:0-1 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731751217321.75a5a39ee7bf008864556f27870615b9. 2024-11-16T10:00:21,800 DEBUG [RS_FLUSH_OPERATIONS-regionserver/037c58a19ddb:0-1 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731751217321.75a5a39ee7bf008864556f27870615b9. as already flushing 2024-11-16T10:00:21,800 DEBUG [RS_FLUSH_OPERATIONS-regionserver/037c58a19ddb:0-1 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731751217321.75a5a39ee7bf008864556f27870615b9. 2024-11-16T10:00:21,800 ERROR [RS_FLUSH_OPERATIONS-regionserver/037c58a19ddb:0-1 {event_type=RS_FLUSH_REGIONS, pid=151}] handler.RSProcedureHandler(58): pid=151 java.io.IOException: Unable to complete flush {ENCODED => 75a5a39ee7bf008864556f27870615b9, NAME => 'TestAcidGuarantees,,1731751217321.75a5a39ee7bf008864556f27870615b9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T10:00:21,800 DEBUG [RS_FLUSH_OPERATIONS-regionserver/037c58a19ddb:0-1 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=151 java.io.IOException: Unable to complete flush {ENCODED => 75a5a39ee7bf008864556f27870615b9, NAME => 'TestAcidGuarantees,,1731751217321.75a5a39ee7bf008864556f27870615b9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T10:00:21,801 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37877 {}] master.HMaster(4177): Remote procedure failed, pid=151 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 75a5a39ee7bf008864556f27870615b9, NAME => 'TestAcidGuarantees,,1731751217321.75a5a39ee7bf008864556f27870615b9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 75a5a39ee7bf008864556f27870615b9, NAME => 'TestAcidGuarantees,,1731751217321.75a5a39ee7bf008864556f27870615b9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T10:00:21,953 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=37267 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=151 2024-11-16T10:00:21,953 DEBUG [RS_FLUSH_OPERATIONS-regionserver/037c58a19ddb:0-2 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731751217321.75a5a39ee7bf008864556f27870615b9. 2024-11-16T10:00:21,953 DEBUG [RS_FLUSH_OPERATIONS-regionserver/037c58a19ddb:0-2 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731751217321.75a5a39ee7bf008864556f27870615b9. as already flushing 2024-11-16T10:00:21,953 DEBUG [RS_FLUSH_OPERATIONS-regionserver/037c58a19ddb:0-2 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731751217321.75a5a39ee7bf008864556f27870615b9. 2024-11-16T10:00:21,953 ERROR [RS_FLUSH_OPERATIONS-regionserver/037c58a19ddb:0-2 {event_type=RS_FLUSH_REGIONS, pid=151}] handler.RSProcedureHandler(58): pid=151 java.io.IOException: Unable to complete flush {ENCODED => 75a5a39ee7bf008864556f27870615b9, NAME => 'TestAcidGuarantees,,1731751217321.75a5a39ee7bf008864556f27870615b9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T10:00:21,953 DEBUG [RS_FLUSH_OPERATIONS-regionserver/037c58a19ddb:0-2 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=151 java.io.IOException: Unable to complete flush {ENCODED => 75a5a39ee7bf008864556f27870615b9, NAME => 'TestAcidGuarantees,,1731751217321.75a5a39ee7bf008864556f27870615b9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T10:00:21,954 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37877 {}] master.HMaster(4177): Remote procedure failed, pid=151 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 75a5a39ee7bf008864556f27870615b9, NAME => 'TestAcidGuarantees,,1731751217321.75a5a39ee7bf008864556f27870615b9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 75a5a39ee7bf008864556f27870615b9, NAME => 'TestAcidGuarantees,,1731751217321.75a5a39ee7bf008864556f27870615b9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T10:00:22,106 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=37267 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=151 2024-11-16T10:00:22,106 DEBUG [RS_FLUSH_OPERATIONS-regionserver/037c58a19ddb:0-0 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731751217321.75a5a39ee7bf008864556f27870615b9. 2024-11-16T10:00:22,106 DEBUG [RS_FLUSH_OPERATIONS-regionserver/037c58a19ddb:0-0 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731751217321.75a5a39ee7bf008864556f27870615b9. as already flushing 2024-11-16T10:00:22,106 DEBUG [RS_FLUSH_OPERATIONS-regionserver/037c58a19ddb:0-0 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731751217321.75a5a39ee7bf008864556f27870615b9. 2024-11-16T10:00:22,106 ERROR [RS_FLUSH_OPERATIONS-regionserver/037c58a19ddb:0-0 {event_type=RS_FLUSH_REGIONS, pid=151}] handler.RSProcedureHandler(58): pid=151 java.io.IOException: Unable to complete flush {ENCODED => 75a5a39ee7bf008864556f27870615b9, NAME => 'TestAcidGuarantees,,1731751217321.75a5a39ee7bf008864556f27870615b9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T10:00:22,106 DEBUG [RS_FLUSH_OPERATIONS-regionserver/037c58a19ddb:0-0 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=151 java.io.IOException: Unable to complete flush {ENCODED => 75a5a39ee7bf008864556f27870615b9, NAME => 'TestAcidGuarantees,,1731751217321.75a5a39ee7bf008864556f27870615b9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T10:00:22,107 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37877 {}] master.HMaster(4177): Remote procedure failed, pid=151 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 75a5a39ee7bf008864556f27870615b9, NAME => 'TestAcidGuarantees,,1731751217321.75a5a39ee7bf008864556f27870615b9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 75a5a39ee7bf008864556f27870615b9, NAME => 'TestAcidGuarantees,,1731751217321.75a5a39ee7bf008864556f27870615b9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T10:00:22,143 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=16 (bloomFilter=true), to=hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/75a5a39ee7bf008864556f27870615b9/.tmp/C/ba3e707513614fbe871b02ce8b163935 2024-11-16T10:00:22,147 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/75a5a39ee7bf008864556f27870615b9/.tmp/A/a08f489d2f324f8c94bc70f75b31e4c0 as hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/75a5a39ee7bf008864556f27870615b9/A/a08f489d2f324f8c94bc70f75b31e4c0 2024-11-16T10:00:22,150 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/75a5a39ee7bf008864556f27870615b9/A/a08f489d2f324f8c94bc70f75b31e4c0, entries=150, sequenceid=16, filesize=30.2 K 2024-11-16T10:00:22,151 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/75a5a39ee7bf008864556f27870615b9/.tmp/B/932ed69b7379451ea7d13e6008653448 as hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/75a5a39ee7bf008864556f27870615b9/B/932ed69b7379451ea7d13e6008653448 2024-11-16T10:00:22,154 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/75a5a39ee7bf008864556f27870615b9/B/932ed69b7379451ea7d13e6008653448, entries=150, sequenceid=16, filesize=11.7 K 2024-11-16T10:00:22,155 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/75a5a39ee7bf008864556f27870615b9/.tmp/C/ba3e707513614fbe871b02ce8b163935 as hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/75a5a39ee7bf008864556f27870615b9/C/ba3e707513614fbe871b02ce8b163935 2024-11-16T10:00:22,158 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/75a5a39ee7bf008864556f27870615b9/C/ba3e707513614fbe871b02ce8b163935, entries=150, sequenceid=16, filesize=11.7 K 2024-11-16T10:00:22,159 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~158.91 KB/162720, currentSize=140.89 KB/144270 for 75a5a39ee7bf008864556f27870615b9 in 1725ms, sequenceid=16, compaction requested=false 2024-11-16T10:00:22,159 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 75a5a39ee7bf008864556f27870615b9: 2024-11-16T10:00:22,258 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=37267 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=151 2024-11-16T10:00:22,259 DEBUG [RS_FLUSH_OPERATIONS-regionserver/037c58a19ddb:0-1 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731751217321.75a5a39ee7bf008864556f27870615b9. 2024-11-16T10:00:22,259 INFO [RS_FLUSH_OPERATIONS-regionserver/037c58a19ddb:0-1 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.HRegion(2902): Flushing 75a5a39ee7bf008864556f27870615b9 3/3 column families, dataSize=140.89 KB heapSize=369.89 KB 2024-11-16T10:00:22,271 DEBUG [RS_FLUSH_OPERATIONS-regionserver/037c58a19ddb:0-1 {event_type=RS_FLUSH_REGIONS, pid=151}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111669b5a8a923f74333a6ac7ef1fa12b14f_75a5a39ee7bf008864556f27870615b9 is 50, key is test_row_0/A:col10/1731751220459/Put/seqid=0 2024-11-16T10:00:22,274 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37591 is added to blk_1073742371_1547 (size=12154) 2024-11-16T10:00:22,274 DEBUG [RS_FLUSH_OPERATIONS-regionserver/037c58a19ddb:0-1 {event_type=RS_FLUSH_REGIONS, pid=151}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:22,278 INFO [RS_FLUSH_OPERATIONS-regionserver/037c58a19ddb:0-1 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111669b5a8a923f74333a6ac7ef1fa12b14f_75a5a39ee7bf008864556f27870615b9 to hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111669b5a8a923f74333a6ac7ef1fa12b14f_75a5a39ee7bf008864556f27870615b9 2024-11-16T10:00:22,279 DEBUG [RS_FLUSH_OPERATIONS-regionserver/037c58a19ddb:0-1 {event_type=RS_FLUSH_REGIONS, pid=151}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/75a5a39ee7bf008864556f27870615b9/.tmp/A/892f7393aaa849a5a4ba84fc0f0c0c2b, store: [table=TestAcidGuarantees family=A region=75a5a39ee7bf008864556f27870615b9] 2024-11-16T10:00:22,279 DEBUG [RS_FLUSH_OPERATIONS-regionserver/037c58a19ddb:0-1 {event_type=RS_FLUSH_REGIONS, pid=151}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/75a5a39ee7bf008864556f27870615b9/.tmp/A/892f7393aaa849a5a4ba84fc0f0c0c2b is 175, key is test_row_0/A:col10/1731751220459/Put/seqid=0 2024-11-16T10:00:22,282 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37591 is added to blk_1073742372_1548 (size=30955) 2024-11-16T10:00:22,283 INFO [RS_FLUSH_OPERATIONS-regionserver/037c58a19ddb:0-1 {event_type=RS_FLUSH_REGIONS, pid=151}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=40, memsize=47.0 K, hasBloomFilter=true, into tmp file hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/75a5a39ee7bf008864556f27870615b9/.tmp/A/892f7393aaa849a5a4ba84fc0f0c0c2b 2024-11-16T10:00:22,290 DEBUG [RS_FLUSH_OPERATIONS-regionserver/037c58a19ddb:0-1 {event_type=RS_FLUSH_REGIONS, pid=151}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/75a5a39ee7bf008864556f27870615b9/.tmp/B/134c29f339e34f62805c458513d26ee8 is 50, key is test_row_0/B:col10/1731751220459/Put/seqid=0 2024-11-16T10:00:22,293 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37591 is added to blk_1073742373_1549 (size=12001) 2024-11-16T10:00:22,293 INFO [RS_FLUSH_OPERATIONS-regionserver/037c58a19ddb:0-1 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=40 (bloomFilter=true), to=hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/75a5a39ee7bf008864556f27870615b9/.tmp/B/134c29f339e34f62805c458513d26ee8 2024-11-16T10:00:22,300 DEBUG [RS_FLUSH_OPERATIONS-regionserver/037c58a19ddb:0-1 {event_type=RS_FLUSH_REGIONS, pid=151}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/75a5a39ee7bf008864556f27870615b9/.tmp/C/ccab6fd1079046d3b9bd608ce597d14a is 50, key is test_row_0/C:col10/1731751220459/Put/seqid=0 2024-11-16T10:00:22,317 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37591 is added to blk_1073742374_1550 (size=12001) 2024-11-16T10:00:22,318 INFO [RS_FLUSH_OPERATIONS-regionserver/037c58a19ddb:0-1 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=40 (bloomFilter=true), to=hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/75a5a39ee7bf008864556f27870615b9/.tmp/C/ccab6fd1079046d3b9bd608ce597d14a 2024-11-16T10:00:22,322 DEBUG [RS_FLUSH_OPERATIONS-regionserver/037c58a19ddb:0-1 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/75a5a39ee7bf008864556f27870615b9/.tmp/A/892f7393aaa849a5a4ba84fc0f0c0c2b as hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/75a5a39ee7bf008864556f27870615b9/A/892f7393aaa849a5a4ba84fc0f0c0c2b 2024-11-16T10:00:22,327 INFO [RS_FLUSH_OPERATIONS-regionserver/037c58a19ddb:0-1 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/75a5a39ee7bf008864556f27870615b9/A/892f7393aaa849a5a4ba84fc0f0c0c2b, entries=150, sequenceid=40, filesize=30.2 K 2024-11-16T10:00:22,328 DEBUG [RS_FLUSH_OPERATIONS-regionserver/037c58a19ddb:0-1 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/75a5a39ee7bf008864556f27870615b9/.tmp/B/134c29f339e34f62805c458513d26ee8 as hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/75a5a39ee7bf008864556f27870615b9/B/134c29f339e34f62805c458513d26ee8 2024-11-16T10:00:22,330 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:22,330 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:22,330 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:22,331 INFO [RS_FLUSH_OPERATIONS-regionserver/037c58a19ddb:0-1 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/75a5a39ee7bf008864556f27870615b9/B/134c29f339e34f62805c458513d26ee8, entries=150, sequenceid=40, filesize=11.7 K 2024-11-16T10:00:22,332 DEBUG [RS_FLUSH_OPERATIONS-regionserver/037c58a19ddb:0-1 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/75a5a39ee7bf008864556f27870615b9/.tmp/C/ccab6fd1079046d3b9bd608ce597d14a as hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/75a5a39ee7bf008864556f27870615b9/C/ccab6fd1079046d3b9bd608ce597d14a 2024-11-16T10:00:22,333 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:22,334 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:22,334 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:22,334 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:22,334 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:22,334 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:22,335 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:22,335 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:22,335 INFO [RS_FLUSH_OPERATIONS-regionserver/037c58a19ddb:0-1 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/75a5a39ee7bf008864556f27870615b9/C/ccab6fd1079046d3b9bd608ce597d14a, entries=150, sequenceid=40, filesize=11.7 K 2024-11-16T10:00:22,335 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:22,335 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:22,336 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:22,336 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:22,336 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:22,336 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:22,336 INFO [RS_FLUSH_OPERATIONS-regionserver/037c58a19ddb:0-1 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.HRegion(3140): Finished flush of dataSize ~140.89 KB/144270, heapSize ~369.84 KB/378720, currentSize=0 B/0 for 75a5a39ee7bf008864556f27870615b9 in 77ms, sequenceid=40, compaction requested=false 2024-11-16T10:00:22,336 DEBUG [RS_FLUSH_OPERATIONS-regionserver/037c58a19ddb:0-1 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.HRegion(2603): Flush status journal for 75a5a39ee7bf008864556f27870615b9: 2024-11-16T10:00:22,336 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:22,336 DEBUG [RS_FLUSH_OPERATIONS-regionserver/037c58a19ddb:0-1 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731751217321.75a5a39ee7bf008864556f27870615b9. 2024-11-16T10:00:22,336 DEBUG [RS_FLUSH_OPERATIONS-regionserver/037c58a19ddb:0-1 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=151 2024-11-16T10:00:22,337 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:22,337 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:22,337 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37877 {}] master.HMaster(4169): Remote procedure done, pid=151 2024-11-16T10:00:22,337 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:22,337 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:22,337 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:22,338 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:22,338 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:22,338 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:22,338 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:22,338 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:22,339 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:22,339 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:22,339 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:22,339 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:22,339 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:22,340 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:22,340 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:22,340 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:22,340 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=151, resume processing ppid=150 2024-11-16T10:00:22,340 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=151, ppid=150, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.9150 sec 2024-11-16T10:00:22,340 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:22,341 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:22,341 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:22,341 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:22,341 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:22,341 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:22,342 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:22,342 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:22,342 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:22,342 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=150, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=150, table=TestAcidGuarantees in 1.9220 sec 2024-11-16T10:00:22,342 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:22,342 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:22,342 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:22,343 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:22,343 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:22,343 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:22,343 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:22,343 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:22,344 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:22,344 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:22,344 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:22,344 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:22,344 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:22,345 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:22,345 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:22,345 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:22,345 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:22,345 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:22,346 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:22,346 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:22,346 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:22,346 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:22,346 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:22,347 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:22,347 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:22,347 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:22,347 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:22,347 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:22,347 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:22,348 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:22,348 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:22,348 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:22,348 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:22,348 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:22,348 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:22,349 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:22,349 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:22,349 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:22,349 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:22,349 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:22,350 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:22,350 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:22,350 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:22,350 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:22,350 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:22,350 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:22,351 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:22,351 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:22,351 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:22,351 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:22,351 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:22,351 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:22,352 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:22,352 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:22,352 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:22,352 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:22,353 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:22,353 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:22,353 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:22,353 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:22,353 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:22,354 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:22,354 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:22,354 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:22,354 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:22,354 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:22,355 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:22,355 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:22,355 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:22,355 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:22,355 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:22,355 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:22,356 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:22,356 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:22,356 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:22,356 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:22,356 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:22,356 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:22,357 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:22,357 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:22,357 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:22,357 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:22,357 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:22,357 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:22,358 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:22,358 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:22,358 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:22,358 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:22,358 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:22,359 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:22,359 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:22,359 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:22,359 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:22,359 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:22,359 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:22,360 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:22,360 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:22,360 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:22,360 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:22,361 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:22,361 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:22,361 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:22,361 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:22,362 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:22,362 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:22,364 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:22,364 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:22,365 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:22,365 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:22,365 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:22,365 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:22,365 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:22,365 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:22,366 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:22,366 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:22,366 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:22,366 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:22,366 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:22,366 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:22,367 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:22,367 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:22,367 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:22,367 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:22,367 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:22,367 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:22,368 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:22,368 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:22,368 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:22,368 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:22,368 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:22,368 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:22,369 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:22,369 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:22,369 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:22,369 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:22,369 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:22,369 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:22,370 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:22,370 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:22,370 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:22,370 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:22,370 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:22,370 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:22,371 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:22,371 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:22,371 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:22,371 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:22,372 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:22,372 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:22,372 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:22,372 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:22,372 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:22,372 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:22,373 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:22,373 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:22,373 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:22,373 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:22,373 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:22,373 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:22,374 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:22,374 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:22,374 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:22,374 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:22,374 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:22,375 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:22,375 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:22,375 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:22,375 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:22,375 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:22,376 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:22,376 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:22,376 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:22,376 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:22,376 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:22,377 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:22,377 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:22,377 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:22,377 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:22,377 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:22,378 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:22,378 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:22,378 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:22,378 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:22,378 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:22,379 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:22,379 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:22,379 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:22,379 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:22,379 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:22,380 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:22,380 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:22,380 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:22,380 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:22,380 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:22,381 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:22,381 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:22,381 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:22,381 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:22,382 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:22,382 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:22,382 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:22,382 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:22,382 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:22,382 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:22,383 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:22,383 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:22,383 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:22,383 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:22,384 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:22,384 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:22,384 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:22,384 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:22,384 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:22,384 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:22,385 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:22,385 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:22,385 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:22,385 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:22,385 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:22,386 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:22,386 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:22,386 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:22,386 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:22,387 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:22,387 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:22,387 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:22,387 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:22,387 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:22,388 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:22,388 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:22,388 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:22,388 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:22,388 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:22,388 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:22,389 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:22,389 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:22,389 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:22,390 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:22,390 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:22,390 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:22,390 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:22,390 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:22,391 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:22,391 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:22,391 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:22,391 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:22,391 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:22,391 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:22,392 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:22,392 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:22,392 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:22,392 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:22,392 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:22,393 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:22,393 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:22,393 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:22,394 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:22,395 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:22,395 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:22,395 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:22,395 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:22,396 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:22,396 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:22,396 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:22,396 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:22,396 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:22,396 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:22,397 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:22,397 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:22,397 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:22,397 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:22,397 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:22,398 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:22,398 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:22,398 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:22,398 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:22,398 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:22,399 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:22,399 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:22,399 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:22,399 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:22,399 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:22,399 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:22,400 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:22,400 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:22,400 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:22,400 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:22,400 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:22,400 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:22,401 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:22,401 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:22,401 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:22,401 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:22,402 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:22,402 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:22,402 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:22,402 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:22,402 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:22,402 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:22,403 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:22,403 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:22,403 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:22,403 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:22,403 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:22,403 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:22,404 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:22,404 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:22,404 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:22,404 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:22,404 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:22,405 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:22,405 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:22,405 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:22,405 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:22,405 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:22,405 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:22,406 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:22,406 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:22,406 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:22,406 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:22,407 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:22,407 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:22,407 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:22,407 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:22,408 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:22,408 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:22,408 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:22,408 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:22,408 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:22,409 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:22,409 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:22,409 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:22,409 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:22,409 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:22,409 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:22,410 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:22,410 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:22,410 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:22,410 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:22,410 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:22,411 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:22,411 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:22,411 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:22,411 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:22,411 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:22,412 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:22,412 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:22,412 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:22,412 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:22,412 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:22,412 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:22,413 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:22,413 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:22,413 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:22,413 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:22,414 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:22,414 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:22,414 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:22,414 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:22,414 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:22,415 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:22,415 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:22,415 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:22,415 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:22,415 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:22,415 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:22,416 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:22,416 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:22,416 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:22,416 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:22,417 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:22,417 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:22,417 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:22,417 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:22,417 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:22,417 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:22,418 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:22,418 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:22,418 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:22,418 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:22,418 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:22,418 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:22,419 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:22,419 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:22,419 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:22,419 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:22,420 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:22,420 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:22,420 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:22,420 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:22,420 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:22,420 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:22,421 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:22,421 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:22,421 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:22,421 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:22,422 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:22,422 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:22,422 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:22,422 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:22,422 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:22,422 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:22,423 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:22,423 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:22,423 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:22,424 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:22,424 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:22,425 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:22,425 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:22,426 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:22,426 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:22,426 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:22,426 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:22,426 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:22,427 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:22,427 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:22,427 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:22,427 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:22,427 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:22,428 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:22,428 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:22,428 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:22,428 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:22,428 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:22,428 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:22,429 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:22,429 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:22,429 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:22,429 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:22,429 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:22,430 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:22,430 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:22,430 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:22,430 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:22,430 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:22,431 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:22,431 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:22,431 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:22,431 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:22,431 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:22,431 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:22,432 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:22,432 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:22,432 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:22,432 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:22,432 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:22,433 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:22,433 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:22,433 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:22,433 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:22,433 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:22,434 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:22,434 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:22,434 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:22,434 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:22,435 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:22,435 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:22,435 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:22,435 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:22,435 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:22,435 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:22,436 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:22,436 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:22,436 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:22,436 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:22,437 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:22,437 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:22,437 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:22,437 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:22,437 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:22,437 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:22,438 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:22,438 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:22,438 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:22,438 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:22,438 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:22,439 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:22,439 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:22,439 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:22,439 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:22,439 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:22,439 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:22,440 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:22,440 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:22,440 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:22,440 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:22,440 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:22,441 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:22,441 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:22,441 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:22,441 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:22,441 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:22,441 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:22,442 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:22,442 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:22,442 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:22,442 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:22,442 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:22,443 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:22,443 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:22,443 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:22,443 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:22,443 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:22,443 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:22,444 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:22,444 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:22,444 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:22,444 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:22,444 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:22,445 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:22,445 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:22,445 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:22,445 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:22,445 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:22,446 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:22,446 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:22,446 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:22,446 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:22,446 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:22,447 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:22,447 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:22,447 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:22,447 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:22,447 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:22,448 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:22,448 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:22,448 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:22,448 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:22,449 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:22,449 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:22,449 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:22,449 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:22,450 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:22,450 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:22,450 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:22,450 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:22,450 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:22,450 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:22,451 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:22,451 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:22,451 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:22,452 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:22,452 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:22,452 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:22,452 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:22,452 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:22,453 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:22,453 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:22,453 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:22,453 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:22,454 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:22,454 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:22,454 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:22,454 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:22,454 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:22,455 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:22,455 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:22,455 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:22,455 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:22,456 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:22,456 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:22,457 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:22,457 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:22,457 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:22,457 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:22,458 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:22,458 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:22,458 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:22,458 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:22,458 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:22,459 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:22,459 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:22,459 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:22,459 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:22,459 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:22,460 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:22,460 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:22,460 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:22,460 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:22,460 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:22,461 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:22,461 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:22,461 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:22,461 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:22,462 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:22,462 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:22,462 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:22,462 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:22,463 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:22,463 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:22,463 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:22,463 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:22,463 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:22,464 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:22,464 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:22,464 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:22,465 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:22,465 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:22,465 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:22,465 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:22,465 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:22,466 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:22,466 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:22,466 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:22,466 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:22,467 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:22,467 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:22,467 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:22,467 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:22,467 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:22,468 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:22,468 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:22,468 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:22,469 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:22,469 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:22,469 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:22,469 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:22,469 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:22,470 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:22,470 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:22,470 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:22,470 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:22,471 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:22,471 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:22,472 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:22,472 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:22,472 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:22,472 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:22,473 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:22,473 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:22,473 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:22,473 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:22,473 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:22,474 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:22,474 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:22,474 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:22,474 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:22,474 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:22,475 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:22,475 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:22,475 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:22,475 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:22,476 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:22,476 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:22,476 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:22,476 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:22,477 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:22,477 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:22,477 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:22,477 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:22,477 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:22,477 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:22,478 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:22,478 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:22,478 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:22,479 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:22,479 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:22,479 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:22,479 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:22,480 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:22,480 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:22,480 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:22,480 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:22,480 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:22,480 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:22,481 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:22,481 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:22,481 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:22,481 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:22,481 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:22,482 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:22,482 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:22,482 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:22,482 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:22,483 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:22,483 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:22,483 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:22,483 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:22,483 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:22,484 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:22,484 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:22,484 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:22,484 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:22,484 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:22,485 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:22,485 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:22,485 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:22,486 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:22,486 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:22,486 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:22,486 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:22,487 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:22,487 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:22,487 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:22,488 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:22,488 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:22,488 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:22,488 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:22,489 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:22,489 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:22,489 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:22,489 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:22,490 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:22,490 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:22,490 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:22,490 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:22,490 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:22,490 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:22,491 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:22,491 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:22,491 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:22,491 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:22,492 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:22,492 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:22,492 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:22,492 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:22,493 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:22,493 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:22,493 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:22,494 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:22,494 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:22,494 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:22,494 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:22,494 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:22,494 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:22,495 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:22,495 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:22,495 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:22,495 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:22,495 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:22,495 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:22,496 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:22,496 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:22,496 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:22,496 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:22,497 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:22,497 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:22,497 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:22,497 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:22,497 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:22,497 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:22,498 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:22,498 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:22,498 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:22,498 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:22,498 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:22,498 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:22,499 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:22,499 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:22,499 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:22,499 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:22,499 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:22,500 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:22,500 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:22,500 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:22,500 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:22,500 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:22,500 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:22,501 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:22,501 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:22,501 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:22,501 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:22,501 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:22,502 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:22,502 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:22,502 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:22,502 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:22,502 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:22,502 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:22,503 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:22,503 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:22,503 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:22,503 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:22,503 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:22,504 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:22,504 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:22,504 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:22,504 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:22,504 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:22,504 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:22,505 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:22,505 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:22,505 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:22,505 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:22,505 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:22,505 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:22,506 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:22,506 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:22,506 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:22,506 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:22,507 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:22,507 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:22,507 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:22,507 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:22,508 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:22,508 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:22,508 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:22,508 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:22,508 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:22,508 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:22,508 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:22,509 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:22,509 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:22,509 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:22,509 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:22,509 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:22,510 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:22,510 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:22,510 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:22,510 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:22,511 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:22,511 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:22,511 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:22,511 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:22,511 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:22,511 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:22,512 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:22,512 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:22,512 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:22,512 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:22,512 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:22,512 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:22,513 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:22,513 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:22,513 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:22,513 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:22,513 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:22,513 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:22,514 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:22,514 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:22,514 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:22,514 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:22,514 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:22,515 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:22,515 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:22,515 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:22,515 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:22,516 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:22,516 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:22,516 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:22,516 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:22,516 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:22,516 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:22,517 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:22,517 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:22,517 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:22,517 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:22,518 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:22,518 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:22,518 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:22,518 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-11-16T10:00:22,518 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:22,519 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:22,519 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:22,519 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:22,520 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:22,520 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:22,520 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:22,520 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:22,521 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:22,521 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:22,521 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:22,521 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:22,522 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:22,522 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:22,522 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:22,523 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:22,523 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:22,523 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:22,523 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:22,524 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:22,524 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:22,524 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:22,524 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:22,525 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:22,525 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:22,525 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:22,525 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:22,525 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:22,526 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:22,526 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:22,526 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:22,526 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:22,527 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:22,527 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:22,527 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:22,527 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:22,528 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:22,528 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:22,528 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:22,529 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:22,529 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:22,529 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:22,529 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:22,529 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:22,530 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:22,530 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:22,530 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:22,530 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:22,530 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:22,531 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:22,531 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:22,531 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:22,532 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:22,532 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:22,532 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:22,533 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:22,533 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:22,533 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:22,533 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:22,534 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:22,534 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:22,534 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:22,535 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:22,535 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:22,535 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:22,535 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:22,535 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:22,535 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:22,536 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:22,536 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:22,536 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:22,536 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:22,536 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:22,536 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:22,537 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:22,538 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:22,538 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:22,538 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:22,539 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:22,539 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:22,539 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:22,539 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:22,539 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:22,539 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:22,540 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:22,540 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:22,540 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:22,541 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:22,541 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:22,541 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:22,541 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:22,542 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:22,542 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:22,542 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:22,542 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:22,542 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:22,543 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:22,543 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:22,543 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:22,543 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:22,544 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:22,544 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:22,544 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:22,544 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:22,544 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:22,545 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:22,545 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:22,545 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:22,545 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:22,545 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:22,546 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:22,546 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:22,546 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:22,546 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:22,547 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:22,547 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:22,547 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:22,547 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:22,547 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:22,548 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:22,548 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:22,548 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:22,548 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:22,549 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:22,549 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:22,549 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:22,549 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:22,549 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:22,549 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:22,550 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:22,550 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:22,550 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:22,550 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:22,550 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:22,551 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:22,551 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:22,551 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:22,551 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:22,551 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:22,551 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:22,551 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:22,552 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:22,552 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:22,552 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:22,553 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:22,553 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:22,553 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:22,553 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:22,553 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:22,553 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:22,554 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:22,554 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:22,554 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:22,555 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:22,555 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:22,555 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:22,555 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:22,556 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:22,556 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:22,556 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:22,557 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:22,557 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:22,557 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:22,557 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:22,557 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:22,558 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:22,558 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:22,558 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:22,558 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:22,559 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:22,559 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:22,560 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:22,560 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:22,560 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:22,560 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:22,560 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:22,561 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:22,561 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:22,561 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:22,561 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:22,561 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:22,562 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:22,562 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:22,562 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:22,562 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:22,562 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:22,563 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:22,563 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:22,563 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:22,564 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:22,564 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:22,564 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:22,564 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:22,565 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:22,565 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:22,565 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:22,565 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:22,565 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:22,566 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:22,566 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:22,566 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:22,567 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:22,567 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:22,567 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:22,567 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:22,567 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:22,568 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:22,568 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:22,568 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:22,568 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:22,569 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:22,569 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:22,569 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:22,569 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:22,569 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:22,570 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:22,570 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:22,570 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:22,571 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:22,571 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:22,571 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:22,571 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:22,571 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:22,572 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:22,572 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:22,572 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:22,572 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:22,573 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:22,573 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:22,573 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37877 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=150 2024-11-16T10:00:22,573 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:22,573 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:22,573 INFO [RPCClient-NioEventLoopGroup-4-12 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-16T10:00:22,573 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:22,574 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:22,574 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37877 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-16T10:00:22,574 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:22,574 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:22,574 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:22,575 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:22,575 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37877 {}] procedure2.ProcedureExecutor(1139): Stored pid=152, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=152, table=TestAcidGuarantees 2024-11-16T10:00:22,575 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:22,575 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:22,575 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:22,575 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37877 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=152 2024-11-16T10:00:22,576 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=152, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=152, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-16T10:00:22,576 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:22,576 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:22,576 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:22,576 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=152, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=152, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-16T10:00:22,577 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=153, ppid=152, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-16T10:00:22,577 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:22,577 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:22,577 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:22,577 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:22,578 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:22,578 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:22,578 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:22,578 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:22,579 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:22,579 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:22,579 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:22,579 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:22,579 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:22,580 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:22,580 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:22,580 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:22,580 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:22,580 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:22,581 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:22,581 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:22,581 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:22,581 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:22,581 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:22,582 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:22,582 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:22,582 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:22,582 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:22,582 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:22,582 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:22,583 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:22,583 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:22,583 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:22,583 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:22,584 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:22,584 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:22,584 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:22,584 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:22,585 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:22,585 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:22,585 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:22,585 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:22,585 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:22,586 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:22,586 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:22,586 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:22,586 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:22,586 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:22,587 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:22,587 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:22,587 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:22,587 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:22,588 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:22,588 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:22,589 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:22,589 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:22,589 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:22,589 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:22,589 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:22,590 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:22,591 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:22,591 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:22,591 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:22,591 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:22,591 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:22,592 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:22,592 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:22,593 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:22,593 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:22,593 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:22,593 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:22,594 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:22,594 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:22,594 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:22,594 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:22,594 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:22,595 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:22,595 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:22,595 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:22,595 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:22,595 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:22,596 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:22,596 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:22,596 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:22,596 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:22,596 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:22,597 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:22,597 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:22,597 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:22,597 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:22,597 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:22,598 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:22,598 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:22,598 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:22,598 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:22,598 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:22,599 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:22,599 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:22,599 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:22,599 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:22,599 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:22,600 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:22,600 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:22,600 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:22,600 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:22,600 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:22,600 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:22,601 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:22,601 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:22,601 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:22,601 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:22,601 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:22,601 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:22,602 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:22,602 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:22,602 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:22,602 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:22,602 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:22,603 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:22,603 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:22,603 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:22,603 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:22,603 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:22,603 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:22,604 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:22,604 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:22,604 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:22,604 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:22,605 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:22,605 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:22,605 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:22,605 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:22,605 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:22,606 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:22,606 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:22,606 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:22,606 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:22,606 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:22,607 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:22,607 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:22,607 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:22,607 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:22,607 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:22,607 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:22,608 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:22,608 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:22,608 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:22,608 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:22,608 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:22,609 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:22,609 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:22,609 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:22,609 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:22,609 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:22,610 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:22,610 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:22,610 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:22,610 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:22,610 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:22,610 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:22,611 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:22,611 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:22,611 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:22,611 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:22,611 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:22,612 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:22,612 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:22,612 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:22,613 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:22,613 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:22,613 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:22,613 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:22,613 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:22,614 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:22,614 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:22,614 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:22,614 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:22,614 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:22,615 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:22,615 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:22,615 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:22,615 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:22,616 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:22,616 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:22,616 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:22,616 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:22,616 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:22,616 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:22,617 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:22,617 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:22,617 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:22,617 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:22,618 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:22,618 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:22,618 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:22,618 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:22,619 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:22,619 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:22,619 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:22,619 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:22,620 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:22,620 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:22,620 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:22,620 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:22,621 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:22,621 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:22,621 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:22,621 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:22,621 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:22,622 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:22,622 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:22,622 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:22,623 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:22,623 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:22,623 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:22,623 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:22,624 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:22,624 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:22,624 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:22,625 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:22,625 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:22,625 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:22,625 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:22,625 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:22,625 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:22,626 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:22,626 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:22,626 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:22,626 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:22,626 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:22,627 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:22,627 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:22,627 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:22,627 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:22,628 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:22,628 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:22,628 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:22,628 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:22,629 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:22,629 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:22,630 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:22,630 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:22,630 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:22,630 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:22,630 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:22,631 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:22,631 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:22,631 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:22,631 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:22,631 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:22,632 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:22,632 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:22,632 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:22,632 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:22,633 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:22,633 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:22,633 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:22,633 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:22,633 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:22,634 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:22,634 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:22,634 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:22,634 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:22,634 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:22,635 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:22,635 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:22,635 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:22,635 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:22,635 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:22,636 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:22,636 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:22,636 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:22,636 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:22,637 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:22,637 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:22,637 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:22,637 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:22,637 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:22,637 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:22,638 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:22,638 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:22,638 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:22,638 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:22,638 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:22,639 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:22,639 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:22,639 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:22,639 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:22,640 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:22,640 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:22,640 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:22,640 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:22,640 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:22,640 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:22,641 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:22,641 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:22,641 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:22,641 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:22,641 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:22,642 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:22,642 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:22,642 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:22,642 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:22,642 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:22,643 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:22,643 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:22,643 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:22,643 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:22,643 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:22,644 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:22,644 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:22,644 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:22,644 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:22,645 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:22,645 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:22,645 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:22,645 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:22,645 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:22,646 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:22,646 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:22,646 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:22,646 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:22,647 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:22,647 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:22,647 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:22,647 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:22,647 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:22,648 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:22,648 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:22,648 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:22,648 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:22,649 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:22,649 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:22,649 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:22,649 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:22,649 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:22,650 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:22,650 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:22,650 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:22,650 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:22,650 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:22,651 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:22,651 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:22,651 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:22,652 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:22,652 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:22,652 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:22,653 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:22,653 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:22,653 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:22,654 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:22,654 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:22,654 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:22,655 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:22,655 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:22,656 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:22,657 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:22,658 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:22,659 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:22,660 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:22,660 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:22,661 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-16T10:00:22,661 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:22,661 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 75a5a39ee7bf008864556f27870615b9 3/3 column families, dataSize=53.67 KB heapSize=141.38 KB 2024-11-16T10:00:22,661 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] regionserver.HRegion(8855): Flush requested on 75a5a39ee7bf008864556f27870615b9 2024-11-16T10:00:22,662 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:22,662 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:22,663 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:22,663 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:22,664 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:22,666 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:22,667 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:22,668 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:22,669 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:22,669 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241116dbbb599b2e134d339c1d04eb23bb4b6f_75a5a39ee7bf008864556f27870615b9 is 50, key is test_row_0/A:col10/1731751222656/Put/seqid=0 2024-11-16T10:00:22,669 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:22,670 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:22,671 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:22,672 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:22,672 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:22,673 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:22,674 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:22,675 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:22,676 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:22,677 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:22,678 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:22,678 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:22,679 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:22,679 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:22,680 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:22,681 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:22,683 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37877 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=152 2024-11-16T10:00:22,686 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:22,687 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:22,688 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:22,688 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:22,689 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:22,690 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:22,690 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:22,691 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:22,692 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:22,692 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:22,693 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:22,694 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:22,694 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:22,695 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:22,695 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:22,696 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:22,697 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:22,698 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:22,699 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:22,699 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:22,700 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:22,701 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37591 is added to blk_1073742376_1552 (size=26798) 2024-11-16T10:00:22,701 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:22,702 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:22,703 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:22,703 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=75a5a39ee7bf008864556f27870615b9, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T10:00:22,704 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34454 deadline: 1731751282696, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=75a5a39ee7bf008864556f27870615b9, server=037c58a19ddb,37267,1731751077844 2024-11-16T10:00:22,704 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:22,704 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=75a5a39ee7bf008864556f27870615b9, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T10:00:22,704 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34430 deadline: 1731751282697, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=75a5a39ee7bf008864556f27870615b9, server=037c58a19ddb,37267,1731751077844 2024-11-16T10:00:22,704 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731751217321.75a5a39ee7bf008864556f27870615b9., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 , the old value is region=TestAcidGuarantees,,1731751217321.75a5a39ee7bf008864556f27870615b9., hostname=037c58a19ddb,37267,1731751077844, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=75a5a39ee7bf008864556f27870615b9, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:00:22,704 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731751217321.75a5a39ee7bf008864556f27870615b9., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=75a5a39ee7bf008864556f27870615b9, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:00:22,704 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731751217321.75a5a39ee7bf008864556f27870615b9., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 because the exception is null or not the one we care about 2024-11-16T10:00:22,705 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:22,705 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731751217321.75a5a39ee7bf008864556f27870615b9., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 , the old value is region=TestAcidGuarantees,,1731751217321.75a5a39ee7bf008864556f27870615b9., hostname=037c58a19ddb,37267,1731751077844, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=75a5a39ee7bf008864556f27870615b9, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:00:22,705 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731751217321.75a5a39ee7bf008864556f27870615b9., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=75a5a39ee7bf008864556f27870615b9, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:00:22,705 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731751217321.75a5a39ee7bf008864556f27870615b9., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 because the exception is null or not the one we care about 2024-11-16T10:00:22,705 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:22,706 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=75a5a39ee7bf008864556f27870615b9, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T10:00:22,706 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34412 deadline: 1731751282699, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=75a5a39ee7bf008864556f27870615b9, server=037c58a19ddb,37267,1731751077844 2024-11-16T10:00:22,707 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731751217321.75a5a39ee7bf008864556f27870615b9., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 , the old value is region=TestAcidGuarantees,,1731751217321.75a5a39ee7bf008864556f27870615b9., hostname=037c58a19ddb,37267,1731751077844, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=75a5a39ee7bf008864556f27870615b9, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:00:22,707 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731751217321.75a5a39ee7bf008864556f27870615b9., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=75a5a39ee7bf008864556f27870615b9, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:00:22,707 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731751217321.75a5a39ee7bf008864556f27870615b9., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 because the exception is null or not the one we care about 2024-11-16T10:00:22,707 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=75a5a39ee7bf008864556f27870615b9, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T10:00:22,707 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34392 deadline: 1731751282702, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=75a5a39ee7bf008864556f27870615b9, server=037c58a19ddb,37267,1731751077844 2024-11-16T10:00:22,707 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731751217321.75a5a39ee7bf008864556f27870615b9., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 , the old value is region=TestAcidGuarantees,,1731751217321.75a5a39ee7bf008864556f27870615b9., hostname=037c58a19ddb,37267,1731751077844, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=75a5a39ee7bf008864556f27870615b9, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:00:22,708 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731751217321.75a5a39ee7bf008864556f27870615b9., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=75a5a39ee7bf008864556f27870615b9, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:00:22,708 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731751217321.75a5a39ee7bf008864556f27870615b9., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 because the exception is null or not the one we care about 2024-11-16T10:00:22,709 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=75a5a39ee7bf008864556f27870615b9, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T10:00:22,709 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34428 deadline: 1731751282704, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=75a5a39ee7bf008864556f27870615b9, server=037c58a19ddb,37267,1731751077844 2024-11-16T10:00:22,709 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731751217321.75a5a39ee7bf008864556f27870615b9., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 , the old value is region=TestAcidGuarantees,,1731751217321.75a5a39ee7bf008864556f27870615b9., hostname=037c58a19ddb,37267,1731751077844, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=75a5a39ee7bf008864556f27870615b9, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:00:22,709 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731751217321.75a5a39ee7bf008864556f27870615b9., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=75a5a39ee7bf008864556f27870615b9, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:00:22,709 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731751217321.75a5a39ee7bf008864556f27870615b9., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 because the exception is null or not the one we care about 2024-11-16T10:00:22,710 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241116dbbb599b2e134d339c1d04eb23bb4b6f_75a5a39ee7bf008864556f27870615b9 to hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241116dbbb599b2e134d339c1d04eb23bb4b6f_75a5a39ee7bf008864556f27870615b9 2024-11-16T10:00:22,711 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/75a5a39ee7bf008864556f27870615b9/.tmp/A/6bc1ee565b014b8782a1e180b4df00da, store: [table=TestAcidGuarantees family=A region=75a5a39ee7bf008864556f27870615b9] 2024-11-16T10:00:22,711 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/75a5a39ee7bf008864556f27870615b9/.tmp/A/6bc1ee565b014b8782a1e180b4df00da is 175, key is test_row_0/A:col10/1731751222656/Put/seqid=0 2024-11-16T10:00:22,714 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37591 is added to blk_1073742375_1551 (size=82584) 2024-11-16T10:00:22,715 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=52, memsize=20.1 K, hasBloomFilter=true, into tmp file hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/75a5a39ee7bf008864556f27870615b9/.tmp/A/6bc1ee565b014b8782a1e180b4df00da 2024-11-16T10:00:22,726 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/75a5a39ee7bf008864556f27870615b9/.tmp/B/bd56f650318741fcbc136f933fd4496c is 50, key is test_row_0/B:col10/1731751222656/Put/seqid=0 2024-11-16T10:00:22,729 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=37267 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=153 2024-11-16T10:00:22,729 DEBUG [RS_FLUSH_OPERATIONS-regionserver/037c58a19ddb:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731751217321.75a5a39ee7bf008864556f27870615b9. 2024-11-16T10:00:22,729 DEBUG [RS_FLUSH_OPERATIONS-regionserver/037c58a19ddb:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731751217321.75a5a39ee7bf008864556f27870615b9. as already flushing 2024-11-16T10:00:22,729 DEBUG [RS_FLUSH_OPERATIONS-regionserver/037c58a19ddb:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731751217321.75a5a39ee7bf008864556f27870615b9. 2024-11-16T10:00:22,729 ERROR [RS_FLUSH_OPERATIONS-regionserver/037c58a19ddb:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] handler.RSProcedureHandler(58): pid=153 java.io.IOException: Unable to complete flush {ENCODED => 75a5a39ee7bf008864556f27870615b9, NAME => 'TestAcidGuarantees,,1731751217321.75a5a39ee7bf008864556f27870615b9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T10:00:22,730 DEBUG [RS_FLUSH_OPERATIONS-regionserver/037c58a19ddb:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=153 java.io.IOException: Unable to complete flush {ENCODED => 75a5a39ee7bf008864556f27870615b9, NAME => 'TestAcidGuarantees,,1731751217321.75a5a39ee7bf008864556f27870615b9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T10:00:22,730 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37877 {}] master.HMaster(4177): Remote procedure failed, pid=153 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 75a5a39ee7bf008864556f27870615b9, NAME => 'TestAcidGuarantees,,1731751217321.75a5a39ee7bf008864556f27870615b9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 75a5a39ee7bf008864556f27870615b9, NAME => 'TestAcidGuarantees,,1731751217321.75a5a39ee7bf008864556f27870615b9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T10:00:22,737 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37591 is added to blk_1073742377_1553 (size=12001) 2024-11-16T10:00:22,814 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=75a5a39ee7bf008864556f27870615b9, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T10:00:22,814 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34454 deadline: 1731751282813, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=75a5a39ee7bf008864556f27870615b9, server=037c58a19ddb,37267,1731751077844 2024-11-16T10:00:22,815 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=75a5a39ee7bf008864556f27870615b9, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T10:00:22,815 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731751217321.75a5a39ee7bf008864556f27870615b9., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 , the old value is region=TestAcidGuarantees,,1731751217321.75a5a39ee7bf008864556f27870615b9., hostname=037c58a19ddb,37267,1731751077844, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=75a5a39ee7bf008864556f27870615b9, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:00:22,815 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731751217321.75a5a39ee7bf008864556f27870615b9., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=75a5a39ee7bf008864556f27870615b9, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:00:22,815 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34430 deadline: 1731751282813, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=75a5a39ee7bf008864556f27870615b9, server=037c58a19ddb,37267,1731751077844 2024-11-16T10:00:22,815 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731751217321.75a5a39ee7bf008864556f27870615b9., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 because the exception is null or not the one we care about 2024-11-16T10:00:22,815 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731751217321.75a5a39ee7bf008864556f27870615b9., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 , the old value is region=TestAcidGuarantees,,1731751217321.75a5a39ee7bf008864556f27870615b9., hostname=037c58a19ddb,37267,1731751077844, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=75a5a39ee7bf008864556f27870615b9, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:00:22,815 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=75a5a39ee7bf008864556f27870615b9, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T10:00:22,815 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731751217321.75a5a39ee7bf008864556f27870615b9., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=75a5a39ee7bf008864556f27870615b9, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:00:22,815 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731751217321.75a5a39ee7bf008864556f27870615b9., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 because the exception is null or not the one we care about 2024-11-16T10:00:22,815 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34412 deadline: 1731751282813, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=75a5a39ee7bf008864556f27870615b9, server=037c58a19ddb,37267,1731751077844 2024-11-16T10:00:22,815 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731751217321.75a5a39ee7bf008864556f27870615b9., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 , the old value is region=TestAcidGuarantees,,1731751217321.75a5a39ee7bf008864556f27870615b9., hostname=037c58a19ddb,37267,1731751077844, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=75a5a39ee7bf008864556f27870615b9, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:00:22,815 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=75a5a39ee7bf008864556f27870615b9, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T10:00:22,815 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731751217321.75a5a39ee7bf008864556f27870615b9., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=75a5a39ee7bf008864556f27870615b9, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:00:22,815 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731751217321.75a5a39ee7bf008864556f27870615b9., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 because the exception is null or not the one we care about 2024-11-16T10:00:22,816 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34392 deadline: 1731751282813, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=75a5a39ee7bf008864556f27870615b9, server=037c58a19ddb,37267,1731751077844 2024-11-16T10:00:22,816 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731751217321.75a5a39ee7bf008864556f27870615b9., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 , the old value is region=TestAcidGuarantees,,1731751217321.75a5a39ee7bf008864556f27870615b9., hostname=037c58a19ddb,37267,1731751077844, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=75a5a39ee7bf008864556f27870615b9, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:00:22,816 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=75a5a39ee7bf008864556f27870615b9, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T10:00:22,816 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731751217321.75a5a39ee7bf008864556f27870615b9., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=75a5a39ee7bf008864556f27870615b9, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:00:22,816 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731751217321.75a5a39ee7bf008864556f27870615b9., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 because the exception is null or not the one we care about 2024-11-16T10:00:22,816 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34428 deadline: 1731751282813, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=75a5a39ee7bf008864556f27870615b9, server=037c58a19ddb,37267,1731751077844 2024-11-16T10:00:22,816 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731751217321.75a5a39ee7bf008864556f27870615b9., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 , the old value is region=TestAcidGuarantees,,1731751217321.75a5a39ee7bf008864556f27870615b9., hostname=037c58a19ddb,37267,1731751077844, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=75a5a39ee7bf008864556f27870615b9, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:00:22,816 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731751217321.75a5a39ee7bf008864556f27870615b9., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=75a5a39ee7bf008864556f27870615b9, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:00:22,816 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731751217321.75a5a39ee7bf008864556f27870615b9., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 because the exception is null or not the one we care about 2024-11-16T10:00:22,883 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=37267 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=153 2024-11-16T10:00:22,883 DEBUG [RS_FLUSH_OPERATIONS-regionserver/037c58a19ddb:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731751217321.75a5a39ee7bf008864556f27870615b9. 2024-11-16T10:00:22,883 DEBUG [RS_FLUSH_OPERATIONS-regionserver/037c58a19ddb:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731751217321.75a5a39ee7bf008864556f27870615b9. as already flushing 2024-11-16T10:00:22,883 DEBUG [RS_FLUSH_OPERATIONS-regionserver/037c58a19ddb:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731751217321.75a5a39ee7bf008864556f27870615b9. 2024-11-16T10:00:22,884 ERROR [RS_FLUSH_OPERATIONS-regionserver/037c58a19ddb:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] handler.RSProcedureHandler(58): pid=153 java.io.IOException: Unable to complete flush {ENCODED => 75a5a39ee7bf008864556f27870615b9, NAME => 'TestAcidGuarantees,,1731751217321.75a5a39ee7bf008864556f27870615b9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T10:00:22,884 DEBUG [RS_FLUSH_OPERATIONS-regionserver/037c58a19ddb:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=153 java.io.IOException: Unable to complete flush {ENCODED => 75a5a39ee7bf008864556f27870615b9, NAME => 'TestAcidGuarantees,,1731751217321.75a5a39ee7bf008864556f27870615b9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T10:00:22,884 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37877 {}] master.HMaster(4177): Remote procedure failed, pid=153 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 75a5a39ee7bf008864556f27870615b9, NAME => 'TestAcidGuarantees,,1731751217321.75a5a39ee7bf008864556f27870615b9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 75a5a39ee7bf008864556f27870615b9, NAME => 'TestAcidGuarantees,,1731751217321.75a5a39ee7bf008864556f27870615b9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T10:00:22,893 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37877 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=152 2024-11-16T10:00:23,026 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=75a5a39ee7bf008864556f27870615b9, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T10:00:23,026 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34454 deadline: 1731751283023, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=75a5a39ee7bf008864556f27870615b9, server=037c58a19ddb,37267,1731751077844 2024-11-16T10:00:23,027 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731751217321.75a5a39ee7bf008864556f27870615b9., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 , the old value is region=TestAcidGuarantees,,1731751217321.75a5a39ee7bf008864556f27870615b9., hostname=037c58a19ddb,37267,1731751077844, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=75a5a39ee7bf008864556f27870615b9, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:00:23,027 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=75a5a39ee7bf008864556f27870615b9, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T10:00:23,027 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731751217321.75a5a39ee7bf008864556f27870615b9., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=75a5a39ee7bf008864556f27870615b9, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:00:23,027 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731751217321.75a5a39ee7bf008864556f27870615b9., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 because the exception is null or not the one we care about 2024-11-16T10:00:23,027 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34430 deadline: 1731751283023, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=75a5a39ee7bf008864556f27870615b9, server=037c58a19ddb,37267,1731751077844 2024-11-16T10:00:23,027 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731751217321.75a5a39ee7bf008864556f27870615b9., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 , the old value is region=TestAcidGuarantees,,1731751217321.75a5a39ee7bf008864556f27870615b9., hostname=037c58a19ddb,37267,1731751077844, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=75a5a39ee7bf008864556f27870615b9, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:00:23,027 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=75a5a39ee7bf008864556f27870615b9, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T10:00:23,027 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731751217321.75a5a39ee7bf008864556f27870615b9., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=75a5a39ee7bf008864556f27870615b9, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:00:23,027 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731751217321.75a5a39ee7bf008864556f27870615b9., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 because the exception is null or not the one we care about 2024-11-16T10:00:23,027 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34412 deadline: 1731751283023, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=75a5a39ee7bf008864556f27870615b9, server=037c58a19ddb,37267,1731751077844 2024-11-16T10:00:23,027 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731751217321.75a5a39ee7bf008864556f27870615b9., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 , the old value is region=TestAcidGuarantees,,1731751217321.75a5a39ee7bf008864556f27870615b9., hostname=037c58a19ddb,37267,1731751077844, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=75a5a39ee7bf008864556f27870615b9, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:00:23,027 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=75a5a39ee7bf008864556f27870615b9, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T10:00:23,027 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731751217321.75a5a39ee7bf008864556f27870615b9., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=75a5a39ee7bf008864556f27870615b9, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:00:23,027 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731751217321.75a5a39ee7bf008864556f27870615b9., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 because the exception is null or not the one we care about 2024-11-16T10:00:23,027 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34392 deadline: 1731751283023, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=75a5a39ee7bf008864556f27870615b9, server=037c58a19ddb,37267,1731751077844 2024-11-16T10:00:23,028 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731751217321.75a5a39ee7bf008864556f27870615b9., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 , the old value is region=TestAcidGuarantees,,1731751217321.75a5a39ee7bf008864556f27870615b9., hostname=037c58a19ddb,37267,1731751077844, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=75a5a39ee7bf008864556f27870615b9, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:00:23,028 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=75a5a39ee7bf008864556f27870615b9, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T10:00:23,028 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731751217321.75a5a39ee7bf008864556f27870615b9., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=75a5a39ee7bf008864556f27870615b9, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:00:23,028 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731751217321.75a5a39ee7bf008864556f27870615b9., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 because the exception is null or not the one we care about 2024-11-16T10:00:23,028 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34428 deadline: 1731751283023, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=75a5a39ee7bf008864556f27870615b9, server=037c58a19ddb,37267,1731751077844 2024-11-16T10:00:23,028 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731751217321.75a5a39ee7bf008864556f27870615b9., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 , the old value is region=TestAcidGuarantees,,1731751217321.75a5a39ee7bf008864556f27870615b9., hostname=037c58a19ddb,37267,1731751077844, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=75a5a39ee7bf008864556f27870615b9, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:00:23,028 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731751217321.75a5a39ee7bf008864556f27870615b9., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=75a5a39ee7bf008864556f27870615b9, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:00:23,028 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731751217321.75a5a39ee7bf008864556f27870615b9., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 because the exception is null or not the one we care about 2024-11-16T10:00:23,036 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=37267 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=153 2024-11-16T10:00:23,036 DEBUG [RS_FLUSH_OPERATIONS-regionserver/037c58a19ddb:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731751217321.75a5a39ee7bf008864556f27870615b9. 2024-11-16T10:00:23,037 DEBUG [RS_FLUSH_OPERATIONS-regionserver/037c58a19ddb:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731751217321.75a5a39ee7bf008864556f27870615b9. as already flushing 2024-11-16T10:00:23,037 DEBUG [RS_FLUSH_OPERATIONS-regionserver/037c58a19ddb:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731751217321.75a5a39ee7bf008864556f27870615b9. 2024-11-16T10:00:23,037 ERROR [RS_FLUSH_OPERATIONS-regionserver/037c58a19ddb:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] handler.RSProcedureHandler(58): pid=153 java.io.IOException: Unable to complete flush {ENCODED => 75a5a39ee7bf008864556f27870615b9, NAME => 'TestAcidGuarantees,,1731751217321.75a5a39ee7bf008864556f27870615b9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T10:00:23,037 DEBUG [RS_FLUSH_OPERATIONS-regionserver/037c58a19ddb:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=153 java.io.IOException: Unable to complete flush {ENCODED => 75a5a39ee7bf008864556f27870615b9, NAME => 'TestAcidGuarantees,,1731751217321.75a5a39ee7bf008864556f27870615b9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T10:00:23,037 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37877 {}] master.HMaster(4177): Remote procedure failed, pid=153 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 75a5a39ee7bf008864556f27870615b9, NAME => 'TestAcidGuarantees,,1731751217321.75a5a39ee7bf008864556f27870615b9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 75a5a39ee7bf008864556f27870615b9, NAME => 'TestAcidGuarantees,,1731751217321.75a5a39ee7bf008864556f27870615b9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T10:00:23,138 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=52 (bloomFilter=true), to=hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/75a5a39ee7bf008864556f27870615b9/.tmp/B/bd56f650318741fcbc136f933fd4496c 2024-11-16T10:00:23,145 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/75a5a39ee7bf008864556f27870615b9/.tmp/C/ac18b2b07c4b45a3a8d847ee65be1577 is 50, key is test_row_0/C:col10/1731751222656/Put/seqid=0 2024-11-16T10:00:23,148 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37591 is added to blk_1073742378_1554 (size=12001) 2024-11-16T10:00:23,189 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=37267 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=153 2024-11-16T10:00:23,189 DEBUG [RS_FLUSH_OPERATIONS-regionserver/037c58a19ddb:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731751217321.75a5a39ee7bf008864556f27870615b9. 2024-11-16T10:00:23,189 DEBUG [RS_FLUSH_OPERATIONS-regionserver/037c58a19ddb:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731751217321.75a5a39ee7bf008864556f27870615b9. as already flushing 2024-11-16T10:00:23,189 DEBUG [RS_FLUSH_OPERATIONS-regionserver/037c58a19ddb:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731751217321.75a5a39ee7bf008864556f27870615b9. 2024-11-16T10:00:23,189 ERROR [RS_FLUSH_OPERATIONS-regionserver/037c58a19ddb:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] handler.RSProcedureHandler(58): pid=153 java.io.IOException: Unable to complete flush {ENCODED => 75a5a39ee7bf008864556f27870615b9, NAME => 'TestAcidGuarantees,,1731751217321.75a5a39ee7bf008864556f27870615b9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T10:00:23,190 DEBUG [RS_FLUSH_OPERATIONS-regionserver/037c58a19ddb:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=153 java.io.IOException: Unable to complete flush {ENCODED => 75a5a39ee7bf008864556f27870615b9, NAME => 'TestAcidGuarantees,,1731751217321.75a5a39ee7bf008864556f27870615b9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T10:00:23,190 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37877 {}] master.HMaster(4177): Remote procedure failed, pid=153 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 75a5a39ee7bf008864556f27870615b9, NAME => 'TestAcidGuarantees,,1731751217321.75a5a39ee7bf008864556f27870615b9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 75a5a39ee7bf008864556f27870615b9, NAME => 'TestAcidGuarantees,,1731751217321.75a5a39ee7bf008864556f27870615b9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T10:00:23,203 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37877 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=152 2024-11-16T10:00:23,336 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=75a5a39ee7bf008864556f27870615b9, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T10:00:23,336 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34454 deadline: 1731751283333, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=75a5a39ee7bf008864556f27870615b9, server=037c58a19ddb,37267,1731751077844 2024-11-16T10:00:23,336 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731751217321.75a5a39ee7bf008864556f27870615b9., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 , the old value is region=TestAcidGuarantees,,1731751217321.75a5a39ee7bf008864556f27870615b9., hostname=037c58a19ddb,37267,1731751077844, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=75a5a39ee7bf008864556f27870615b9, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:00:23,336 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=75a5a39ee7bf008864556f27870615b9, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T10:00:23,336 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731751217321.75a5a39ee7bf008864556f27870615b9., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=75a5a39ee7bf008864556f27870615b9, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:00:23,336 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731751217321.75a5a39ee7bf008864556f27870615b9., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 because the exception is null or not the one we care about 2024-11-16T10:00:23,336 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34412 deadline: 1731751283333, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=75a5a39ee7bf008864556f27870615b9, server=037c58a19ddb,37267,1731751077844 2024-11-16T10:00:23,337 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731751217321.75a5a39ee7bf008864556f27870615b9., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 , the old value is region=TestAcidGuarantees,,1731751217321.75a5a39ee7bf008864556f27870615b9., hostname=037c58a19ddb,37267,1731751077844, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=75a5a39ee7bf008864556f27870615b9, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:00:23,337 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731751217321.75a5a39ee7bf008864556f27870615b9., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=75a5a39ee7bf008864556f27870615b9, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:00:23,337 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731751217321.75a5a39ee7bf008864556f27870615b9., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 because the exception is null or not the one we care about 2024-11-16T10:00:23,337 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=75a5a39ee7bf008864556f27870615b9, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T10:00:23,337 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34430 deadline: 1731751283333, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=75a5a39ee7bf008864556f27870615b9, server=037c58a19ddb,37267,1731751077844 2024-11-16T10:00:23,337 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731751217321.75a5a39ee7bf008864556f27870615b9., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 , the old value is region=TestAcidGuarantees,,1731751217321.75a5a39ee7bf008864556f27870615b9., hostname=037c58a19ddb,37267,1731751077844, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=75a5a39ee7bf008864556f27870615b9, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:00:23,337 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=75a5a39ee7bf008864556f27870615b9, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T10:00:23,337 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=75a5a39ee7bf008864556f27870615b9, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T10:00:23,337 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731751217321.75a5a39ee7bf008864556f27870615b9., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=75a5a39ee7bf008864556f27870615b9, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:00:23,337 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731751217321.75a5a39ee7bf008864556f27870615b9., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 because the exception is null or not the one we care about 2024-11-16T10:00:23,337 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34428 deadline: 1731751283333, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=75a5a39ee7bf008864556f27870615b9, server=037c58a19ddb,37267,1731751077844 2024-11-16T10:00:23,337 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34392 deadline: 1731751283333, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=75a5a39ee7bf008864556f27870615b9, server=037c58a19ddb,37267,1731751077844 2024-11-16T10:00:23,337 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731751217321.75a5a39ee7bf008864556f27870615b9., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 , the old value is region=TestAcidGuarantees,,1731751217321.75a5a39ee7bf008864556f27870615b9., hostname=037c58a19ddb,37267,1731751077844, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=75a5a39ee7bf008864556f27870615b9, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:00:23,337 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731751217321.75a5a39ee7bf008864556f27870615b9., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=75a5a39ee7bf008864556f27870615b9, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:00:23,338 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731751217321.75a5a39ee7bf008864556f27870615b9., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 because the exception is null or not the one we care about 2024-11-16T10:00:23,338 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731751217321.75a5a39ee7bf008864556f27870615b9., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 , the old value is region=TestAcidGuarantees,,1731751217321.75a5a39ee7bf008864556f27870615b9., hostname=037c58a19ddb,37267,1731751077844, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=75a5a39ee7bf008864556f27870615b9, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:00:23,338 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731751217321.75a5a39ee7bf008864556f27870615b9., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=75a5a39ee7bf008864556f27870615b9, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:00:23,338 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731751217321.75a5a39ee7bf008864556f27870615b9., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 because the exception is null or not the one we care about 2024-11-16T10:00:23,342 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=37267 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=153 2024-11-16T10:00:23,342 DEBUG [RS_FLUSH_OPERATIONS-regionserver/037c58a19ddb:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731751217321.75a5a39ee7bf008864556f27870615b9. 2024-11-16T10:00:23,342 DEBUG [RS_FLUSH_OPERATIONS-regionserver/037c58a19ddb:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731751217321.75a5a39ee7bf008864556f27870615b9. as already flushing 2024-11-16T10:00:23,342 DEBUG [RS_FLUSH_OPERATIONS-regionserver/037c58a19ddb:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731751217321.75a5a39ee7bf008864556f27870615b9. 2024-11-16T10:00:23,342 ERROR [RS_FLUSH_OPERATIONS-regionserver/037c58a19ddb:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] handler.RSProcedureHandler(58): pid=153 java.io.IOException: Unable to complete flush {ENCODED => 75a5a39ee7bf008864556f27870615b9, NAME => 'TestAcidGuarantees,,1731751217321.75a5a39ee7bf008864556f27870615b9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T10:00:23,342 DEBUG [RS_FLUSH_OPERATIONS-regionserver/037c58a19ddb:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=153 java.io.IOException: Unable to complete flush {ENCODED => 75a5a39ee7bf008864556f27870615b9, NAME => 'TestAcidGuarantees,,1731751217321.75a5a39ee7bf008864556f27870615b9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T10:00:23,343 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37877 {}] master.HMaster(4177): Remote procedure failed, pid=153 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 75a5a39ee7bf008864556f27870615b9, NAME => 'TestAcidGuarantees,,1731751217321.75a5a39ee7bf008864556f27870615b9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 75a5a39ee7bf008864556f27870615b9, NAME => 'TestAcidGuarantees,,1731751217321.75a5a39ee7bf008864556f27870615b9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T10:00:23,495 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=37267 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=153 2024-11-16T10:00:23,495 DEBUG [RS_FLUSH_OPERATIONS-regionserver/037c58a19ddb:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731751217321.75a5a39ee7bf008864556f27870615b9. 2024-11-16T10:00:23,495 DEBUG [RS_FLUSH_OPERATIONS-regionserver/037c58a19ddb:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731751217321.75a5a39ee7bf008864556f27870615b9. as already flushing 2024-11-16T10:00:23,495 DEBUG [RS_FLUSH_OPERATIONS-regionserver/037c58a19ddb:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731751217321.75a5a39ee7bf008864556f27870615b9. 2024-11-16T10:00:23,495 ERROR [RS_FLUSH_OPERATIONS-regionserver/037c58a19ddb:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] handler.RSProcedureHandler(58): pid=153 java.io.IOException: Unable to complete flush {ENCODED => 75a5a39ee7bf008864556f27870615b9, NAME => 'TestAcidGuarantees,,1731751217321.75a5a39ee7bf008864556f27870615b9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T10:00:23,495 DEBUG [RS_FLUSH_OPERATIONS-regionserver/037c58a19ddb:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=153 java.io.IOException: Unable to complete flush {ENCODED => 75a5a39ee7bf008864556f27870615b9, NAME => 'TestAcidGuarantees,,1731751217321.75a5a39ee7bf008864556f27870615b9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T10:00:23,496 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37877 {}] master.HMaster(4177): Remote procedure failed, pid=153 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 75a5a39ee7bf008864556f27870615b9, NAME => 'TestAcidGuarantees,,1731751217321.75a5a39ee7bf008864556f27870615b9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 75a5a39ee7bf008864556f27870615b9, NAME => 'TestAcidGuarantees,,1731751217321.75a5a39ee7bf008864556f27870615b9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T10:00:23,549 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=52 (bloomFilter=true), to=hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/75a5a39ee7bf008864556f27870615b9/.tmp/C/ac18b2b07c4b45a3a8d847ee65be1577 2024-11-16T10:00:23,553 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/75a5a39ee7bf008864556f27870615b9/.tmp/A/6bc1ee565b014b8782a1e180b4df00da as hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/75a5a39ee7bf008864556f27870615b9/A/6bc1ee565b014b8782a1e180b4df00da 2024-11-16T10:00:23,556 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/75a5a39ee7bf008864556f27870615b9/A/6bc1ee565b014b8782a1e180b4df00da, entries=450, sequenceid=52, filesize=80.6 K 2024-11-16T10:00:23,557 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/75a5a39ee7bf008864556f27870615b9/.tmp/B/bd56f650318741fcbc136f933fd4496c as hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/75a5a39ee7bf008864556f27870615b9/B/bd56f650318741fcbc136f933fd4496c 2024-11-16T10:00:23,560 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/75a5a39ee7bf008864556f27870615b9/B/bd56f650318741fcbc136f933fd4496c, entries=150, sequenceid=52, filesize=11.7 K 2024-11-16T10:00:23,560 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/75a5a39ee7bf008864556f27870615b9/.tmp/C/ac18b2b07c4b45a3a8d847ee65be1577 as hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/75a5a39ee7bf008864556f27870615b9/C/ac18b2b07c4b45a3a8d847ee65be1577 2024-11-16T10:00:23,564 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/75a5a39ee7bf008864556f27870615b9/C/ac18b2b07c4b45a3a8d847ee65be1577, entries=150, sequenceid=52, filesize=11.7 K 2024-11-16T10:00:23,565 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~158.91 KB/162720, currentSize=140.89 KB/144270 for 75a5a39ee7bf008864556f27870615b9 in 903ms, sequenceid=52, compaction requested=true 2024-11-16T10:00:23,565 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 75a5a39ee7bf008864556f27870615b9: 2024-11-16T10:00:23,565 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 75a5a39ee7bf008864556f27870615b9:A, priority=-2147483648, current under compaction store size is 1 2024-11-16T10:00:23,565 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-16T10:00:23,565 DEBUG [RS:0;037c58a19ddb:37267-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-16T10:00:23,565 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 75a5a39ee7bf008864556f27870615b9:B, priority=-2147483648, current under compaction store size is 2 2024-11-16T10:00:23,565 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-16T10:00:23,565 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 75a5a39ee7bf008864556f27870615b9:C, priority=-2147483648, current under compaction store size is 3 2024-11-16T10:00:23,565 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-16T10:00:23,565 DEBUG [RS:0;037c58a19ddb:37267-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-16T10:00:23,566 DEBUG [RS:0;037c58a19ddb:37267-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 144494 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-16T10:00:23,566 DEBUG [RS:0;037c58a19ddb:37267-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36003 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-16T10:00:23,566 DEBUG [RS:0;037c58a19ddb:37267-longCompactions-0 {}] regionserver.HStore(1541): 75a5a39ee7bf008864556f27870615b9/B is initiating minor compaction (all files) 2024-11-16T10:00:23,566 DEBUG [RS:0;037c58a19ddb:37267-shortCompactions-0 {}] regionserver.HStore(1541): 75a5a39ee7bf008864556f27870615b9/A is initiating minor compaction (all files) 2024-11-16T10:00:23,566 INFO [RS:0;037c58a19ddb:37267-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 75a5a39ee7bf008864556f27870615b9/B in TestAcidGuarantees,,1731751217321.75a5a39ee7bf008864556f27870615b9. 2024-11-16T10:00:23,566 INFO [RS:0;037c58a19ddb:37267-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 75a5a39ee7bf008864556f27870615b9/A in TestAcidGuarantees,,1731751217321.75a5a39ee7bf008864556f27870615b9. 2024-11-16T10:00:23,566 INFO [RS:0;037c58a19ddb:37267-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/75a5a39ee7bf008864556f27870615b9/B/932ed69b7379451ea7d13e6008653448, hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/75a5a39ee7bf008864556f27870615b9/B/134c29f339e34f62805c458513d26ee8, hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/75a5a39ee7bf008864556f27870615b9/B/bd56f650318741fcbc136f933fd4496c] into tmpdir=hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/75a5a39ee7bf008864556f27870615b9/.tmp, totalSize=35.2 K 2024-11-16T10:00:23,566 INFO [RS:0;037c58a19ddb:37267-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/75a5a39ee7bf008864556f27870615b9/A/a08f489d2f324f8c94bc70f75b31e4c0, hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/75a5a39ee7bf008864556f27870615b9/A/892f7393aaa849a5a4ba84fc0f0c0c2b, hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/75a5a39ee7bf008864556f27870615b9/A/6bc1ee565b014b8782a1e180b4df00da] into tmpdir=hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/75a5a39ee7bf008864556f27870615b9/.tmp, totalSize=141.1 K 2024-11-16T10:00:23,566 INFO [RS:0;037c58a19ddb:37267-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731751217321.75a5a39ee7bf008864556f27870615b9. 2024-11-16T10:00:23,566 DEBUG [RS:0;037c58a19ddb:37267-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731751217321.75a5a39ee7bf008864556f27870615b9. files: [hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/75a5a39ee7bf008864556f27870615b9/A/a08f489d2f324f8c94bc70f75b31e4c0, hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/75a5a39ee7bf008864556f27870615b9/A/892f7393aaa849a5a4ba84fc0f0c0c2b, hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/75a5a39ee7bf008864556f27870615b9/A/6bc1ee565b014b8782a1e180b4df00da] 2024-11-16T10:00:23,567 DEBUG [RS:0;037c58a19ddb:37267-longCompactions-0 {}] compactions.Compactor(225): Compacting 932ed69b7379451ea7d13e6008653448, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=16, earliestPutTs=1731751220427 2024-11-16T10:00:23,567 DEBUG [RS:0;037c58a19ddb:37267-shortCompactions-0 {}] compactions.Compactor(225): Compacting a08f489d2f324f8c94bc70f75b31e4c0, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=16, earliestPutTs=1731751220427 2024-11-16T10:00:23,567 DEBUG [RS:0;037c58a19ddb:37267-longCompactions-0 {}] compactions.Compactor(225): Compacting 134c29f339e34f62805c458513d26ee8, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=40, earliestPutTs=1731751220459 2024-11-16T10:00:23,567 DEBUG [RS:0;037c58a19ddb:37267-shortCompactions-0 {}] compactions.Compactor(225): Compacting 892f7393aaa849a5a4ba84fc0f0c0c2b, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=40, earliestPutTs=1731751220459 2024-11-16T10:00:23,567 DEBUG [RS:0;037c58a19ddb:37267-longCompactions-0 {}] compactions.Compactor(225): Compacting bd56f650318741fcbc136f933fd4496c, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=52, earliestPutTs=1731751222656 2024-11-16T10:00:23,567 DEBUG [RS:0;037c58a19ddb:37267-shortCompactions-0 {}] compactions.Compactor(225): Compacting 6bc1ee565b014b8782a1e180b4df00da, keycount=450, bloomtype=ROW, size=80.6 K, encoding=NONE, compression=NONE, seqNum=52, earliestPutTs=1731751222628 2024-11-16T10:00:23,572 INFO [RS:0;037c58a19ddb:37267-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=75a5a39ee7bf008864556f27870615b9] 2024-11-16T10:00:23,573 INFO [RS:0;037c58a19ddb:37267-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 75a5a39ee7bf008864556f27870615b9#B#compaction#475 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-16T10:00:23,574 DEBUG [RS:0;037c58a19ddb:37267-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/75a5a39ee7bf008864556f27870615b9/.tmp/B/2a05f259717c4e27b74a376fd6d40bf9 is 50, key is test_row_0/B:col10/1731751222656/Put/seqid=0 2024-11-16T10:00:23,584 DEBUG [RS:0;037c58a19ddb:37267-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e202411160dfdc3fc579c4f738295bccb8193cb86_75a5a39ee7bf008864556f27870615b9 store=[table=TestAcidGuarantees family=A region=75a5a39ee7bf008864556f27870615b9] 2024-11-16T10:00:23,587 DEBUG [RS:0;037c58a19ddb:37267-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e202411160dfdc3fc579c4f738295bccb8193cb86_75a5a39ee7bf008864556f27870615b9, store=[table=TestAcidGuarantees family=A region=75a5a39ee7bf008864556f27870615b9] 2024-11-16T10:00:23,588 DEBUG [RS:0;037c58a19ddb:37267-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411160dfdc3fc579c4f738295bccb8193cb86_75a5a39ee7bf008864556f27870615b9 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=75a5a39ee7bf008864556f27870615b9] 2024-11-16T10:00:23,597 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37591 is added to blk_1073742379_1555 (size=12104) 2024-11-16T10:00:23,609 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37591 is added to blk_1073742380_1556 (size=4469) 2024-11-16T10:00:23,648 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=37267 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=153 2024-11-16T10:00:23,648 DEBUG [RS_FLUSH_OPERATIONS-regionserver/037c58a19ddb:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731751217321.75a5a39ee7bf008864556f27870615b9. 2024-11-16T10:00:23,648 INFO [RS_FLUSH_OPERATIONS-regionserver/037c58a19ddb:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.HRegion(2902): Flushing 75a5a39ee7bf008864556f27870615b9 3/3 column families, dataSize=140.89 KB heapSize=369.89 KB 2024-11-16T10:00:23,655 DEBUG [RS_FLUSH_OPERATIONS-regionserver/037c58a19ddb:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111699da5f2a2dd244dfa3fb0f6d31790c49_75a5a39ee7bf008864556f27870615b9 is 50, key is test_row_0/A:col10/1731751222690/Put/seqid=0 2024-11-16T10:00:23,658 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37591 is added to blk_1073742381_1557 (size=12154) 2024-11-16T10:00:23,712 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37877 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=152 2024-11-16T10:00:23,847 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] regionserver.HRegion(8855): Flush requested on 75a5a39ee7bf008864556f27870615b9 2024-11-16T10:00:23,847 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731751217321.75a5a39ee7bf008864556f27870615b9. as already flushing 2024-11-16T10:00:23,859 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=75a5a39ee7bf008864556f27870615b9, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T10:00:23,860 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34454 deadline: 1731751283854, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=75a5a39ee7bf008864556f27870615b9, server=037c58a19ddb,37267,1731751077844 2024-11-16T10:00:23,860 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731751217321.75a5a39ee7bf008864556f27870615b9., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 , the old value is region=TestAcidGuarantees,,1731751217321.75a5a39ee7bf008864556f27870615b9., hostname=037c58a19ddb,37267,1731751077844, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=75a5a39ee7bf008864556f27870615b9, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:00:23,860 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731751217321.75a5a39ee7bf008864556f27870615b9., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=75a5a39ee7bf008864556f27870615b9, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:00:23,860 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731751217321.75a5a39ee7bf008864556f27870615b9., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 because the exception is null or not the one we care about 2024-11-16T10:00:23,860 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=75a5a39ee7bf008864556f27870615b9, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T10:00:23,860 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34412 deadline: 1731751283855, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=75a5a39ee7bf008864556f27870615b9, server=037c58a19ddb,37267,1731751077844 2024-11-16T10:00:23,861 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731751217321.75a5a39ee7bf008864556f27870615b9., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 , the old value is region=TestAcidGuarantees,,1731751217321.75a5a39ee7bf008864556f27870615b9., hostname=037c58a19ddb,37267,1731751077844, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=75a5a39ee7bf008864556f27870615b9, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:00:23,861 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731751217321.75a5a39ee7bf008864556f27870615b9., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=75a5a39ee7bf008864556f27870615b9, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:00:23,861 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=75a5a39ee7bf008864556f27870615b9, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T10:00:23,861 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731751217321.75a5a39ee7bf008864556f27870615b9., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 because the exception is null or not the one we care about 2024-11-16T10:00:23,861 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34392 deadline: 1731751283855, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=75a5a39ee7bf008864556f27870615b9, server=037c58a19ddb,37267,1731751077844 2024-11-16T10:00:23,861 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731751217321.75a5a39ee7bf008864556f27870615b9., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 , the old value is region=TestAcidGuarantees,,1731751217321.75a5a39ee7bf008864556f27870615b9., hostname=037c58a19ddb,37267,1731751077844, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=75a5a39ee7bf008864556f27870615b9, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:00:23,861 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731751217321.75a5a39ee7bf008864556f27870615b9., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=75a5a39ee7bf008864556f27870615b9, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:00:23,861 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731751217321.75a5a39ee7bf008864556f27870615b9., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 because the exception is null or not the one we care about 2024-11-16T10:00:23,863 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=75a5a39ee7bf008864556f27870615b9, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T10:00:23,863 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34428 deadline: 1731751283860, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=75a5a39ee7bf008864556f27870615b9, server=037c58a19ddb,37267,1731751077844 2024-11-16T10:00:23,863 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731751217321.75a5a39ee7bf008864556f27870615b9., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 , the old value is region=TestAcidGuarantees,,1731751217321.75a5a39ee7bf008864556f27870615b9., hostname=037c58a19ddb,37267,1731751077844, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=75a5a39ee7bf008864556f27870615b9, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:00:23,863 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731751217321.75a5a39ee7bf008864556f27870615b9., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=75a5a39ee7bf008864556f27870615b9, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:00:23,863 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731751217321.75a5a39ee7bf008864556f27870615b9., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 because the exception is null or not the one we care about 2024-11-16T10:00:23,866 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=75a5a39ee7bf008864556f27870615b9, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T10:00:23,866 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34430 deadline: 1731751283860, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=75a5a39ee7bf008864556f27870615b9, server=037c58a19ddb,37267,1731751077844 2024-11-16T10:00:23,866 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731751217321.75a5a39ee7bf008864556f27870615b9., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 , the old value is region=TestAcidGuarantees,,1731751217321.75a5a39ee7bf008864556f27870615b9., hostname=037c58a19ddb,37267,1731751077844, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=75a5a39ee7bf008864556f27870615b9, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:00:23,866 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731751217321.75a5a39ee7bf008864556f27870615b9., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=75a5a39ee7bf008864556f27870615b9, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:00:23,866 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731751217321.75a5a39ee7bf008864556f27870615b9., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 because the exception is null or not the one we care about 2024-11-16T10:00:23,967 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=75a5a39ee7bf008864556f27870615b9, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T10:00:23,967 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34412 deadline: 1731751283963, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=75a5a39ee7bf008864556f27870615b9, server=037c58a19ddb,37267,1731751077844 2024-11-16T10:00:23,967 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=75a5a39ee7bf008864556f27870615b9, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T10:00:23,967 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731751217321.75a5a39ee7bf008864556f27870615b9., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 , the old value is region=TestAcidGuarantees,,1731751217321.75a5a39ee7bf008864556f27870615b9., hostname=037c58a19ddb,37267,1731751077844, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=75a5a39ee7bf008864556f27870615b9, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:00:23,967 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34454 deadline: 1731751283963, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=75a5a39ee7bf008864556f27870615b9, server=037c58a19ddb,37267,1731751077844 2024-11-16T10:00:23,967 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731751217321.75a5a39ee7bf008864556f27870615b9., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=75a5a39ee7bf008864556f27870615b9, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:00:23,967 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731751217321.75a5a39ee7bf008864556f27870615b9., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 because the exception is null or not the one we care about 2024-11-16T10:00:23,967 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731751217321.75a5a39ee7bf008864556f27870615b9., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 , the old value is region=TestAcidGuarantees,,1731751217321.75a5a39ee7bf008864556f27870615b9., hostname=037c58a19ddb,37267,1731751077844, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=75a5a39ee7bf008864556f27870615b9, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:00:23,967 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731751217321.75a5a39ee7bf008864556f27870615b9., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=75a5a39ee7bf008864556f27870615b9, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:00:23,967 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731751217321.75a5a39ee7bf008864556f27870615b9., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 because the exception is null or not the one we care about 2024-11-16T10:00:23,968 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=75a5a39ee7bf008864556f27870615b9, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T10:00:23,968 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34392 deadline: 1731751283964, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=75a5a39ee7bf008864556f27870615b9, server=037c58a19ddb,37267,1731751077844 2024-11-16T10:00:23,968 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731751217321.75a5a39ee7bf008864556f27870615b9., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 , the old value is region=TestAcidGuarantees,,1731751217321.75a5a39ee7bf008864556f27870615b9., hostname=037c58a19ddb,37267,1731751077844, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=75a5a39ee7bf008864556f27870615b9, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:00:23,968 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731751217321.75a5a39ee7bf008864556f27870615b9., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=75a5a39ee7bf008864556f27870615b9, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:00:23,968 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731751217321.75a5a39ee7bf008864556f27870615b9., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 because the exception is null or not the one we care about 2024-11-16T10:00:23,976 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=75a5a39ee7bf008864556f27870615b9, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T10:00:23,976 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34430 deadline: 1731751283972, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=75a5a39ee7bf008864556f27870615b9, server=037c58a19ddb,37267,1731751077844 2024-11-16T10:00:23,976 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731751217321.75a5a39ee7bf008864556f27870615b9., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 , the old value is region=TestAcidGuarantees,,1731751217321.75a5a39ee7bf008864556f27870615b9., hostname=037c58a19ddb,37267,1731751077844, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=75a5a39ee7bf008864556f27870615b9, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:00:23,976 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=75a5a39ee7bf008864556f27870615b9, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T10:00:23,976 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731751217321.75a5a39ee7bf008864556f27870615b9., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=75a5a39ee7bf008864556f27870615b9, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:00:23,976 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731751217321.75a5a39ee7bf008864556f27870615b9., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 because the exception is null or not the one we care about 2024-11-16T10:00:23,976 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34428 deadline: 1731751283972, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=75a5a39ee7bf008864556f27870615b9, server=037c58a19ddb,37267,1731751077844 2024-11-16T10:00:23,976 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731751217321.75a5a39ee7bf008864556f27870615b9., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 , the old value is region=TestAcidGuarantees,,1731751217321.75a5a39ee7bf008864556f27870615b9., hostname=037c58a19ddb,37267,1731751077844, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=75a5a39ee7bf008864556f27870615b9, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:00:23,976 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731751217321.75a5a39ee7bf008864556f27870615b9., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=75a5a39ee7bf008864556f27870615b9, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:00:23,976 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731751217321.75a5a39ee7bf008864556f27870615b9., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 because the exception is null or not the one we care about 2024-11-16T10:00:24,001 DEBUG [RS:0;037c58a19ddb:37267-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/75a5a39ee7bf008864556f27870615b9/.tmp/B/2a05f259717c4e27b74a376fd6d40bf9 as hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/75a5a39ee7bf008864556f27870615b9/B/2a05f259717c4e27b74a376fd6d40bf9 2024-11-16T10:00:24,006 INFO [RS:0;037c58a19ddb:37267-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 75a5a39ee7bf008864556f27870615b9/B of 75a5a39ee7bf008864556f27870615b9 into 2a05f259717c4e27b74a376fd6d40bf9(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-16T10:00:24,006 DEBUG [RS:0;037c58a19ddb:37267-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 75a5a39ee7bf008864556f27870615b9: 2024-11-16T10:00:24,006 INFO [RS:0;037c58a19ddb:37267-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731751217321.75a5a39ee7bf008864556f27870615b9., storeName=75a5a39ee7bf008864556f27870615b9/B, priority=13, startTime=1731751223565; duration=0sec 2024-11-16T10:00:24,006 DEBUG [RS:0;037c58a19ddb:37267-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-16T10:00:24,006 DEBUG [RS:0;037c58a19ddb:37267-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 75a5a39ee7bf008864556f27870615b9:B 2024-11-16T10:00:24,006 DEBUG [RS:0;037c58a19ddb:37267-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-16T10:00:24,008 DEBUG [RS:0;037c58a19ddb:37267-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36003 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-16T10:00:24,008 DEBUG [RS:0;037c58a19ddb:37267-longCompactions-0 {}] regionserver.HStore(1541): 75a5a39ee7bf008864556f27870615b9/C is initiating minor compaction (all files) 2024-11-16T10:00:24,008 INFO [RS:0;037c58a19ddb:37267-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 75a5a39ee7bf008864556f27870615b9/C in TestAcidGuarantees,,1731751217321.75a5a39ee7bf008864556f27870615b9. 2024-11-16T10:00:24,008 INFO [RS:0;037c58a19ddb:37267-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/75a5a39ee7bf008864556f27870615b9/C/ba3e707513614fbe871b02ce8b163935, hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/75a5a39ee7bf008864556f27870615b9/C/ccab6fd1079046d3b9bd608ce597d14a, hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/75a5a39ee7bf008864556f27870615b9/C/ac18b2b07c4b45a3a8d847ee65be1577] into tmpdir=hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/75a5a39ee7bf008864556f27870615b9/.tmp, totalSize=35.2 K 2024-11-16T10:00:24,008 DEBUG [RS:0;037c58a19ddb:37267-longCompactions-0 {}] compactions.Compactor(225): Compacting ba3e707513614fbe871b02ce8b163935, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=16, earliestPutTs=1731751220427 2024-11-16T10:00:24,009 DEBUG [RS:0;037c58a19ddb:37267-longCompactions-0 {}] compactions.Compactor(225): Compacting ccab6fd1079046d3b9bd608ce597d14a, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=40, earliestPutTs=1731751220459 2024-11-16T10:00:24,010 DEBUG [RS:0;037c58a19ddb:37267-longCompactions-0 {}] compactions.Compactor(225): Compacting ac18b2b07c4b45a3a8d847ee65be1577, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=52, earliestPutTs=1731751222656 2024-11-16T10:00:24,010 INFO [RS:0;037c58a19ddb:37267-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 75a5a39ee7bf008864556f27870615b9#A#compaction#476 average throughput is 0.06 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-16T10:00:24,011 DEBUG [RS:0;037c58a19ddb:37267-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/75a5a39ee7bf008864556f27870615b9/.tmp/A/56efa3c087bc425a838c68ca08f65887 is 175, key is test_row_0/A:col10/1731751222656/Put/seqid=0 2024-11-16T10:00:24,019 INFO [RS:0;037c58a19ddb:37267-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 75a5a39ee7bf008864556f27870615b9#C#compaction#478 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-16T10:00:24,020 DEBUG [RS:0;037c58a19ddb:37267-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/75a5a39ee7bf008864556f27870615b9/.tmp/C/660a6b8d351a4709a39d1668e126c1e3 is 50, key is test_row_0/C:col10/1731751222656/Put/seqid=0 2024-11-16T10:00:24,028 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37591 is added to blk_1073742382_1558 (size=31058) 2024-11-16T10:00:24,039 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37591 is added to blk_1073742383_1559 (size=12104) 2024-11-16T10:00:24,060 DEBUG [RS_FLUSH_OPERATIONS-regionserver/037c58a19ddb:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:24,064 INFO [RS_FLUSH_OPERATIONS-regionserver/037c58a19ddb:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111699da5f2a2dd244dfa3fb0f6d31790c49_75a5a39ee7bf008864556f27870615b9 to hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111699da5f2a2dd244dfa3fb0f6d31790c49_75a5a39ee7bf008864556f27870615b9 2024-11-16T10:00:24,065 DEBUG [RS_FLUSH_OPERATIONS-regionserver/037c58a19ddb:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/75a5a39ee7bf008864556f27870615b9/.tmp/A/9712b27b0e4f495e9650ab7a025b0ba1, store: [table=TestAcidGuarantees family=A region=75a5a39ee7bf008864556f27870615b9] 2024-11-16T10:00:24,066 DEBUG [RS_FLUSH_OPERATIONS-regionserver/037c58a19ddb:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/75a5a39ee7bf008864556f27870615b9/.tmp/A/9712b27b0e4f495e9650ab7a025b0ba1 is 175, key is test_row_0/A:col10/1731751222690/Put/seqid=0 2024-11-16T10:00:24,098 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37591 is added to blk_1073742384_1560 (size=30955) 2024-11-16T10:00:24,099 INFO [RS_FLUSH_OPERATIONS-regionserver/037c58a19ddb:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=76, memsize=47.0 K, hasBloomFilter=true, into tmp file hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/75a5a39ee7bf008864556f27870615b9/.tmp/A/9712b27b0e4f495e9650ab7a025b0ba1 2024-11-16T10:00:24,110 DEBUG [RS_FLUSH_OPERATIONS-regionserver/037c58a19ddb:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/75a5a39ee7bf008864556f27870615b9/.tmp/B/3807f7330657475a8b56368c81de0fe8 is 50, key is test_row_0/B:col10/1731751222690/Put/seqid=0 2024-11-16T10:00:24,122 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37591 is added to blk_1073742385_1561 (size=12001) 2024-11-16T10:00:24,177 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=75a5a39ee7bf008864556f27870615b9, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T10:00:24,177 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34454 deadline: 1731751284173, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=75a5a39ee7bf008864556f27870615b9, server=037c58a19ddb,37267,1731751077844 2024-11-16T10:00:24,177 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=75a5a39ee7bf008864556f27870615b9, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T10:00:24,177 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34392 deadline: 1731751284173, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=75a5a39ee7bf008864556f27870615b9, server=037c58a19ddb,37267,1731751077844 2024-11-16T10:00:24,177 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731751217321.75a5a39ee7bf008864556f27870615b9., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 , the old value is region=TestAcidGuarantees,,1731751217321.75a5a39ee7bf008864556f27870615b9., hostname=037c58a19ddb,37267,1731751077844, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=75a5a39ee7bf008864556f27870615b9, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:00:24,177 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731751217321.75a5a39ee7bf008864556f27870615b9., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=75a5a39ee7bf008864556f27870615b9, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:00:24,177 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731751217321.75a5a39ee7bf008864556f27870615b9., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 because the exception is null or not the one we care about 2024-11-16T10:00:24,177 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=75a5a39ee7bf008864556f27870615b9, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T10:00:24,177 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731751217321.75a5a39ee7bf008864556f27870615b9., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 , the old value is region=TestAcidGuarantees,,1731751217321.75a5a39ee7bf008864556f27870615b9., hostname=037c58a19ddb,37267,1731751077844, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=75a5a39ee7bf008864556f27870615b9, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:00:24,177 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731751217321.75a5a39ee7bf008864556f27870615b9., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=75a5a39ee7bf008864556f27870615b9, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:00:24,177 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731751217321.75a5a39ee7bf008864556f27870615b9., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 because the exception is null or not the one we care about 2024-11-16T10:00:24,177 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34412 deadline: 1731751284174, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=75a5a39ee7bf008864556f27870615b9, server=037c58a19ddb,37267,1731751077844 2024-11-16T10:00:24,178 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731751217321.75a5a39ee7bf008864556f27870615b9., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 , the old value is region=TestAcidGuarantees,,1731751217321.75a5a39ee7bf008864556f27870615b9., hostname=037c58a19ddb,37267,1731751077844, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=75a5a39ee7bf008864556f27870615b9, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:00:24,178 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731751217321.75a5a39ee7bf008864556f27870615b9., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=75a5a39ee7bf008864556f27870615b9, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:00:24,178 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731751217321.75a5a39ee7bf008864556f27870615b9., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 because the exception is null or not the one we care about 2024-11-16T10:00:24,185 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=75a5a39ee7bf008864556f27870615b9, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T10:00:24,186 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34428 deadline: 1731751284183, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=75a5a39ee7bf008864556f27870615b9, server=037c58a19ddb,37267,1731751077844 2024-11-16T10:00:24,186 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=75a5a39ee7bf008864556f27870615b9, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T10:00:24,186 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34430 deadline: 1731751284183, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=75a5a39ee7bf008864556f27870615b9, server=037c58a19ddb,37267,1731751077844 2024-11-16T10:00:24,186 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731751217321.75a5a39ee7bf008864556f27870615b9., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 , the old value is region=TestAcidGuarantees,,1731751217321.75a5a39ee7bf008864556f27870615b9., hostname=037c58a19ddb,37267,1731751077844, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=75a5a39ee7bf008864556f27870615b9, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:00:24,186 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731751217321.75a5a39ee7bf008864556f27870615b9., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=75a5a39ee7bf008864556f27870615b9, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:00:24,186 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731751217321.75a5a39ee7bf008864556f27870615b9., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 because the exception is null or not the one we care about 2024-11-16T10:00:24,186 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731751217321.75a5a39ee7bf008864556f27870615b9., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 , the old value is region=TestAcidGuarantees,,1731751217321.75a5a39ee7bf008864556f27870615b9., hostname=037c58a19ddb,37267,1731751077844, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=75a5a39ee7bf008864556f27870615b9, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:00:24,186 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731751217321.75a5a39ee7bf008864556f27870615b9., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=75a5a39ee7bf008864556f27870615b9, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:00:24,186 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731751217321.75a5a39ee7bf008864556f27870615b9., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 because the exception is null or not the one we care about 2024-11-16T10:00:24,433 DEBUG [RS:0;037c58a19ddb:37267-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/75a5a39ee7bf008864556f27870615b9/.tmp/A/56efa3c087bc425a838c68ca08f65887 as hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/75a5a39ee7bf008864556f27870615b9/A/56efa3c087bc425a838c68ca08f65887 2024-11-16T10:00:24,438 INFO [RS:0;037c58a19ddb:37267-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 75a5a39ee7bf008864556f27870615b9/A of 75a5a39ee7bf008864556f27870615b9 into 56efa3c087bc425a838c68ca08f65887(size=30.3 K), total size for store is 30.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-16T10:00:24,438 DEBUG [RS:0;037c58a19ddb:37267-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 75a5a39ee7bf008864556f27870615b9: 2024-11-16T10:00:24,438 INFO [RS:0;037c58a19ddb:37267-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731751217321.75a5a39ee7bf008864556f27870615b9., storeName=75a5a39ee7bf008864556f27870615b9/A, priority=13, startTime=1731751223565; duration=0sec 2024-11-16T10:00:24,438 DEBUG [RS:0;037c58a19ddb:37267-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-16T10:00:24,438 DEBUG [RS:0;037c58a19ddb:37267-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 75a5a39ee7bf008864556f27870615b9:A 2024-11-16T10:00:24,443 DEBUG [RS:0;037c58a19ddb:37267-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/75a5a39ee7bf008864556f27870615b9/.tmp/C/660a6b8d351a4709a39d1668e126c1e3 as hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/75a5a39ee7bf008864556f27870615b9/C/660a6b8d351a4709a39d1668e126c1e3 2024-11-16T10:00:24,447 INFO [RS:0;037c58a19ddb:37267-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 75a5a39ee7bf008864556f27870615b9/C of 75a5a39ee7bf008864556f27870615b9 into 660a6b8d351a4709a39d1668e126c1e3(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-16T10:00:24,447 DEBUG [RS:0;037c58a19ddb:37267-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 75a5a39ee7bf008864556f27870615b9: 2024-11-16T10:00:24,447 INFO [RS:0;037c58a19ddb:37267-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731751217321.75a5a39ee7bf008864556f27870615b9., storeName=75a5a39ee7bf008864556f27870615b9/C, priority=13, startTime=1731751223565; duration=0sec 2024-11-16T10:00:24,448 DEBUG [RS:0;037c58a19ddb:37267-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-16T10:00:24,448 DEBUG [RS:0;037c58a19ddb:37267-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 75a5a39ee7bf008864556f27870615b9:C 2024-11-16T10:00:24,485 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=75a5a39ee7bf008864556f27870615b9, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T10:00:24,486 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34454 deadline: 1731751284483, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=75a5a39ee7bf008864556f27870615b9, server=037c58a19ddb,37267,1731751077844 2024-11-16T10:00:24,486 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731751217321.75a5a39ee7bf008864556f27870615b9., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 , the old value is region=TestAcidGuarantees,,1731751217321.75a5a39ee7bf008864556f27870615b9., hostname=037c58a19ddb,37267,1731751077844, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=75a5a39ee7bf008864556f27870615b9, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:00:24,486 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=75a5a39ee7bf008864556f27870615b9, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T10:00:24,486 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731751217321.75a5a39ee7bf008864556f27870615b9., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=75a5a39ee7bf008864556f27870615b9, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:00:24,486 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731751217321.75a5a39ee7bf008864556f27870615b9., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 because the exception is null or not the one we care about 2024-11-16T10:00:24,486 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34412 deadline: 1731751284483, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=75a5a39ee7bf008864556f27870615b9, server=037c58a19ddb,37267,1731751077844 2024-11-16T10:00:24,486 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=75a5a39ee7bf008864556f27870615b9, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T10:00:24,486 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34392 deadline: 1731751284483, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=75a5a39ee7bf008864556f27870615b9, server=037c58a19ddb,37267,1731751077844 2024-11-16T10:00:24,486 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731751217321.75a5a39ee7bf008864556f27870615b9., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 , the old value is region=TestAcidGuarantees,,1731751217321.75a5a39ee7bf008864556f27870615b9., hostname=037c58a19ddb,37267,1731751077844, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=75a5a39ee7bf008864556f27870615b9, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:00:24,486 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731751217321.75a5a39ee7bf008864556f27870615b9., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=75a5a39ee7bf008864556f27870615b9, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:00:24,486 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731751217321.75a5a39ee7bf008864556f27870615b9., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 because the exception is null or not the one we care about 2024-11-16T10:00:24,486 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731751217321.75a5a39ee7bf008864556f27870615b9., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 , the old value is region=TestAcidGuarantees,,1731751217321.75a5a39ee7bf008864556f27870615b9., hostname=037c58a19ddb,37267,1731751077844, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=75a5a39ee7bf008864556f27870615b9, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:00:24,487 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731751217321.75a5a39ee7bf008864556f27870615b9., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=75a5a39ee7bf008864556f27870615b9, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:00:24,487 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731751217321.75a5a39ee7bf008864556f27870615b9., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 because the exception is null or not the one we care about 2024-11-16T10:00:24,494 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=75a5a39ee7bf008864556f27870615b9, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T10:00:24,494 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34428 deadline: 1731751284492, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=75a5a39ee7bf008864556f27870615b9, server=037c58a19ddb,37267,1731751077844 2024-11-16T10:00:24,494 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=75a5a39ee7bf008864556f27870615b9, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T10:00:24,494 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34430 deadline: 1731751284492, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=75a5a39ee7bf008864556f27870615b9, server=037c58a19ddb,37267,1731751077844 2024-11-16T10:00:24,494 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731751217321.75a5a39ee7bf008864556f27870615b9., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 , the old value is region=TestAcidGuarantees,,1731751217321.75a5a39ee7bf008864556f27870615b9., hostname=037c58a19ddb,37267,1731751077844, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=75a5a39ee7bf008864556f27870615b9, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:00:24,494 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731751217321.75a5a39ee7bf008864556f27870615b9., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=75a5a39ee7bf008864556f27870615b9, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:00:24,494 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731751217321.75a5a39ee7bf008864556f27870615b9., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 because the exception is null or not the one we care about 2024-11-16T10:00:24,495 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731751217321.75a5a39ee7bf008864556f27870615b9., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 , the old value is region=TestAcidGuarantees,,1731751217321.75a5a39ee7bf008864556f27870615b9., hostname=037c58a19ddb,37267,1731751077844, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=75a5a39ee7bf008864556f27870615b9, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:00:24,495 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731751217321.75a5a39ee7bf008864556f27870615b9., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=75a5a39ee7bf008864556f27870615b9, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:00:24,495 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731751217321.75a5a39ee7bf008864556f27870615b9., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 because the exception is null or not the one we care about 2024-11-16T10:00:24,522 INFO [RS_FLUSH_OPERATIONS-regionserver/037c58a19ddb:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=76 (bloomFilter=true), to=hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/75a5a39ee7bf008864556f27870615b9/.tmp/B/3807f7330657475a8b56368c81de0fe8 2024-11-16T10:00:24,529 DEBUG [RS_FLUSH_OPERATIONS-regionserver/037c58a19ddb:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/75a5a39ee7bf008864556f27870615b9/.tmp/C/4634faf075a148e0aa55658f0b57d51f is 50, key is test_row_0/C:col10/1731751222690/Put/seqid=0 2024-11-16T10:00:24,535 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37591 is added to blk_1073742386_1562 (size=12001) 2024-11-16T10:00:24,723 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37877 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=152 2024-11-16T10:00:24,935 INFO [RS_FLUSH_OPERATIONS-regionserver/037c58a19ddb:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=76 (bloomFilter=true), to=hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/75a5a39ee7bf008864556f27870615b9/.tmp/C/4634faf075a148e0aa55658f0b57d51f 2024-11-16T10:00:24,939 DEBUG [RS_FLUSH_OPERATIONS-regionserver/037c58a19ddb:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/75a5a39ee7bf008864556f27870615b9/.tmp/A/9712b27b0e4f495e9650ab7a025b0ba1 as hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/75a5a39ee7bf008864556f27870615b9/A/9712b27b0e4f495e9650ab7a025b0ba1 2024-11-16T10:00:24,943 INFO [RS_FLUSH_OPERATIONS-regionserver/037c58a19ddb:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/75a5a39ee7bf008864556f27870615b9/A/9712b27b0e4f495e9650ab7a025b0ba1, entries=150, sequenceid=76, filesize=30.2 K 2024-11-16T10:00:24,943 DEBUG [RS_FLUSH_OPERATIONS-regionserver/037c58a19ddb:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/75a5a39ee7bf008864556f27870615b9/.tmp/B/3807f7330657475a8b56368c81de0fe8 as hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/75a5a39ee7bf008864556f27870615b9/B/3807f7330657475a8b56368c81de0fe8 2024-11-16T10:00:24,947 INFO [RS_FLUSH_OPERATIONS-regionserver/037c58a19ddb:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/75a5a39ee7bf008864556f27870615b9/B/3807f7330657475a8b56368c81de0fe8, entries=150, sequenceid=76, filesize=11.7 K 2024-11-16T10:00:24,947 DEBUG [RS_FLUSH_OPERATIONS-regionserver/037c58a19ddb:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/75a5a39ee7bf008864556f27870615b9/.tmp/C/4634faf075a148e0aa55658f0b57d51f as hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/75a5a39ee7bf008864556f27870615b9/C/4634faf075a148e0aa55658f0b57d51f 2024-11-16T10:00:24,957 INFO [RS_FLUSH_OPERATIONS-regionserver/037c58a19ddb:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/75a5a39ee7bf008864556f27870615b9/C/4634faf075a148e0aa55658f0b57d51f, entries=150, sequenceid=76, filesize=11.7 K 2024-11-16T10:00:24,958 INFO [RS_FLUSH_OPERATIONS-regionserver/037c58a19ddb:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.HRegion(3140): Finished flush of dataSize ~140.89 KB/144270, heapSize ~369.84 KB/378720, currentSize=67.09 KB/68700 for 75a5a39ee7bf008864556f27870615b9 in 1310ms, sequenceid=76, compaction requested=false 2024-11-16T10:00:24,958 DEBUG [RS_FLUSH_OPERATIONS-regionserver/037c58a19ddb:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.HRegion(2603): Flush status journal for 75a5a39ee7bf008864556f27870615b9: 2024-11-16T10:00:24,958 DEBUG [RS_FLUSH_OPERATIONS-regionserver/037c58a19ddb:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731751217321.75a5a39ee7bf008864556f27870615b9. 2024-11-16T10:00:24,958 DEBUG [RS_FLUSH_OPERATIONS-regionserver/037c58a19ddb:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=153 2024-11-16T10:00:24,959 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37877 {}] master.HMaster(4169): Remote procedure done, pid=153 2024-11-16T10:00:24,961 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=153, resume processing ppid=152 2024-11-16T10:00:24,961 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=153, ppid=152, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.3840 sec 2024-11-16T10:00:24,963 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=152, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=152, table=TestAcidGuarantees in 2.3880 sec 2024-11-16T10:00:24,996 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] regionserver.HRegion(8855): Flush requested on 75a5a39ee7bf008864556f27870615b9 2024-11-16T10:00:24,996 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-16T10:00:24,996 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 75a5a39ee7bf008864556f27870615b9 3/3 column families, dataSize=80.51 KB heapSize=211.69 KB 2024-11-16T10:00:25,003 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241116a4d23bce912c47b1accd9d147df295f7_75a5a39ee7bf008864556f27870615b9 is 50, key is test_row_0/A:col10/1731751224995/Put/seqid=0 2024-11-16T10:00:25,007 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37591 is added to blk_1073742387_1563 (size=14594) 2024-11-16T10:00:25,023 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=75a5a39ee7bf008864556f27870615b9, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T10:00:25,023 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34428 deadline: 1731751285017, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=75a5a39ee7bf008864556f27870615b9, server=037c58a19ddb,37267,1731751077844 2024-11-16T10:00:25,023 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731751217321.75a5a39ee7bf008864556f27870615b9., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 , the old value is region=TestAcidGuarantees,,1731751217321.75a5a39ee7bf008864556f27870615b9., hostname=037c58a19ddb,37267,1731751077844, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=75a5a39ee7bf008864556f27870615b9, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:00:25,023 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731751217321.75a5a39ee7bf008864556f27870615b9., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=75a5a39ee7bf008864556f27870615b9, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:00:25,023 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731751217321.75a5a39ee7bf008864556f27870615b9., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 because the exception is null or not the one we care about 2024-11-16T10:00:25,027 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=75a5a39ee7bf008864556f27870615b9, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T10:00:25,027 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=75a5a39ee7bf008864556f27870615b9, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T10:00:25,027 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34454 deadline: 1731751285022, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=75a5a39ee7bf008864556f27870615b9, server=037c58a19ddb,37267,1731751077844 2024-11-16T10:00:25,027 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34412 deadline: 1731751285022, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=75a5a39ee7bf008864556f27870615b9, server=037c58a19ddb,37267,1731751077844 2024-11-16T10:00:25,027 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=75a5a39ee7bf008864556f27870615b9, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T10:00:25,027 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731751217321.75a5a39ee7bf008864556f27870615b9., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 , the old value is region=TestAcidGuarantees,,1731751217321.75a5a39ee7bf008864556f27870615b9., hostname=037c58a19ddb,37267,1731751077844, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=75a5a39ee7bf008864556f27870615b9, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:00:25,027 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=75a5a39ee7bf008864556f27870615b9, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T10:00:25,027 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731751217321.75a5a39ee7bf008864556f27870615b9., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=75a5a39ee7bf008864556f27870615b9, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:00:25,027 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34430 deadline: 1731751285023, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=75a5a39ee7bf008864556f27870615b9, server=037c58a19ddb,37267,1731751077844 2024-11-16T10:00:25,027 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731751217321.75a5a39ee7bf008864556f27870615b9., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 because the exception is null or not the one we care about 2024-11-16T10:00:25,027 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34392 deadline: 1731751285023, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=75a5a39ee7bf008864556f27870615b9, server=037c58a19ddb,37267,1731751077844 2024-11-16T10:00:25,027 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731751217321.75a5a39ee7bf008864556f27870615b9., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 , the old value is region=TestAcidGuarantees,,1731751217321.75a5a39ee7bf008864556f27870615b9., hostname=037c58a19ddb,37267,1731751077844, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=75a5a39ee7bf008864556f27870615b9, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:00:25,027 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731751217321.75a5a39ee7bf008864556f27870615b9., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=75a5a39ee7bf008864556f27870615b9, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:00:25,027 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731751217321.75a5a39ee7bf008864556f27870615b9., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 because the exception is null or not the one we care about 2024-11-16T10:00:25,028 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731751217321.75a5a39ee7bf008864556f27870615b9., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 , the old value is region=TestAcidGuarantees,,1731751217321.75a5a39ee7bf008864556f27870615b9., hostname=037c58a19ddb,37267,1731751077844, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=75a5a39ee7bf008864556f27870615b9, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:00:25,028 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731751217321.75a5a39ee7bf008864556f27870615b9., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=75a5a39ee7bf008864556f27870615b9, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:00:25,028 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731751217321.75a5a39ee7bf008864556f27870615b9., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 , the old value is region=TestAcidGuarantees,,1731751217321.75a5a39ee7bf008864556f27870615b9., hostname=037c58a19ddb,37267,1731751077844, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=75a5a39ee7bf008864556f27870615b9, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:00:25,028 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731751217321.75a5a39ee7bf008864556f27870615b9., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 because the exception is null or not the one we care about 2024-11-16T10:00:25,028 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731751217321.75a5a39ee7bf008864556f27870615b9., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=75a5a39ee7bf008864556f27870615b9, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:00:25,028 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731751217321.75a5a39ee7bf008864556f27870615b9., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 because the exception is null or not the one we care about 2024-11-16T10:00:25,135 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=75a5a39ee7bf008864556f27870615b9, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T10:00:25,136 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34428 deadline: 1731751285133, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=75a5a39ee7bf008864556f27870615b9, server=037c58a19ddb,37267,1731751077844 2024-11-16T10:00:25,136 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=75a5a39ee7bf008864556f27870615b9, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T10:00:25,136 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731751217321.75a5a39ee7bf008864556f27870615b9., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 , the old value is region=TestAcidGuarantees,,1731751217321.75a5a39ee7bf008864556f27870615b9., hostname=037c58a19ddb,37267,1731751077844, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=75a5a39ee7bf008864556f27870615b9, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:00:25,136 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731751217321.75a5a39ee7bf008864556f27870615b9., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=75a5a39ee7bf008864556f27870615b9, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:00:25,136 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731751217321.75a5a39ee7bf008864556f27870615b9., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 because the exception is null or not the one we care about 2024-11-16T10:00:25,136 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34454 deadline: 1731751285133, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=75a5a39ee7bf008864556f27870615b9, server=037c58a19ddb,37267,1731751077844 2024-11-16T10:00:25,136 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=75a5a39ee7bf008864556f27870615b9, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T10:00:25,136 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731751217321.75a5a39ee7bf008864556f27870615b9., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 , the old value is region=TestAcidGuarantees,,1731751217321.75a5a39ee7bf008864556f27870615b9., hostname=037c58a19ddb,37267,1731751077844, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=75a5a39ee7bf008864556f27870615b9, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:00:25,136 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34412 deadline: 1731751285133, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=75a5a39ee7bf008864556f27870615b9, server=037c58a19ddb,37267,1731751077844 2024-11-16T10:00:25,136 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731751217321.75a5a39ee7bf008864556f27870615b9., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=75a5a39ee7bf008864556f27870615b9, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:00:25,136 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731751217321.75a5a39ee7bf008864556f27870615b9., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 because the exception is null or not the one we care about 2024-11-16T10:00:25,136 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=75a5a39ee7bf008864556f27870615b9, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T10:00:25,136 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34430 deadline: 1731751285133, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=75a5a39ee7bf008864556f27870615b9, server=037c58a19ddb,37267,1731751077844 2024-11-16T10:00:25,136 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731751217321.75a5a39ee7bf008864556f27870615b9., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 , the old value is region=TestAcidGuarantees,,1731751217321.75a5a39ee7bf008864556f27870615b9., hostname=037c58a19ddb,37267,1731751077844, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=75a5a39ee7bf008864556f27870615b9, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:00:25,137 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731751217321.75a5a39ee7bf008864556f27870615b9., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=75a5a39ee7bf008864556f27870615b9, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:00:25,137 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731751217321.75a5a39ee7bf008864556f27870615b9., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 because the exception is null or not the one we care about 2024-11-16T10:00:25,137 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=75a5a39ee7bf008864556f27870615b9, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T10:00:25,137 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731751217321.75a5a39ee7bf008864556f27870615b9., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 , the old value is region=TestAcidGuarantees,,1731751217321.75a5a39ee7bf008864556f27870615b9., hostname=037c58a19ddb,37267,1731751077844, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=75a5a39ee7bf008864556f27870615b9, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:00:25,137 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731751217321.75a5a39ee7bf008864556f27870615b9., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=75a5a39ee7bf008864556f27870615b9, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:00:25,137 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34392 deadline: 1731751285133, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=75a5a39ee7bf008864556f27870615b9, server=037c58a19ddb,37267,1731751077844 2024-11-16T10:00:25,137 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731751217321.75a5a39ee7bf008864556f27870615b9., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 because the exception is null or not the one we care about 2024-11-16T10:00:25,137 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731751217321.75a5a39ee7bf008864556f27870615b9., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 , the old value is region=TestAcidGuarantees,,1731751217321.75a5a39ee7bf008864556f27870615b9., hostname=037c58a19ddb,37267,1731751077844, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=75a5a39ee7bf008864556f27870615b9, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:00:25,137 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731751217321.75a5a39ee7bf008864556f27870615b9., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=75a5a39ee7bf008864556f27870615b9, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:00:25,137 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731751217321.75a5a39ee7bf008864556f27870615b9., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 because the exception is null or not the one we care about 2024-11-16T10:00:25,344 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=75a5a39ee7bf008864556f27870615b9, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T10:00:25,344 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34428 deadline: 1731751285343, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=75a5a39ee7bf008864556f27870615b9, server=037c58a19ddb,37267,1731751077844 2024-11-16T10:00:25,344 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=75a5a39ee7bf008864556f27870615b9, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T10:00:25,344 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731751217321.75a5a39ee7bf008864556f27870615b9., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 , the old value is region=TestAcidGuarantees,,1731751217321.75a5a39ee7bf008864556f27870615b9., hostname=037c58a19ddb,37267,1731751077844, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=75a5a39ee7bf008864556f27870615b9, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:00:25,344 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34454 deadline: 1731751285343, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=75a5a39ee7bf008864556f27870615b9, server=037c58a19ddb,37267,1731751077844 2024-11-16T10:00:25,344 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731751217321.75a5a39ee7bf008864556f27870615b9., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=75a5a39ee7bf008864556f27870615b9, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:00:25,344 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731751217321.75a5a39ee7bf008864556f27870615b9., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 because the exception is null or not the one we care about 2024-11-16T10:00:25,345 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=75a5a39ee7bf008864556f27870615b9, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T10:00:25,345 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34412 deadline: 1731751285343, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=75a5a39ee7bf008864556f27870615b9, server=037c58a19ddb,37267,1731751077844 2024-11-16T10:00:25,345 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731751217321.75a5a39ee7bf008864556f27870615b9., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 , the old value is region=TestAcidGuarantees,,1731751217321.75a5a39ee7bf008864556f27870615b9., hostname=037c58a19ddb,37267,1731751077844, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=75a5a39ee7bf008864556f27870615b9, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:00:25,345 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731751217321.75a5a39ee7bf008864556f27870615b9., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=75a5a39ee7bf008864556f27870615b9, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:00:25,345 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731751217321.75a5a39ee7bf008864556f27870615b9., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 because the exception is null or not the one we care about 2024-11-16T10:00:25,345 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=75a5a39ee7bf008864556f27870615b9, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T10:00:25,345 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731751217321.75a5a39ee7bf008864556f27870615b9., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 , the old value is region=TestAcidGuarantees,,1731751217321.75a5a39ee7bf008864556f27870615b9., hostname=037c58a19ddb,37267,1731751077844, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=75a5a39ee7bf008864556f27870615b9, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:00:25,345 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731751217321.75a5a39ee7bf008864556f27870615b9., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=75a5a39ee7bf008864556f27870615b9, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:00:25,345 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34430 deadline: 1731751285343, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=75a5a39ee7bf008864556f27870615b9, server=037c58a19ddb,37267,1731751077844 2024-11-16T10:00:25,345 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731751217321.75a5a39ee7bf008864556f27870615b9., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 because the exception is null or not the one we care about 2024-11-16T10:00:25,345 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=75a5a39ee7bf008864556f27870615b9, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T10:00:25,345 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731751217321.75a5a39ee7bf008864556f27870615b9., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 , the old value is region=TestAcidGuarantees,,1731751217321.75a5a39ee7bf008864556f27870615b9., hostname=037c58a19ddb,37267,1731751077844, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=75a5a39ee7bf008864556f27870615b9, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:00:25,345 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34392 deadline: 1731751285343, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=75a5a39ee7bf008864556f27870615b9, server=037c58a19ddb,37267,1731751077844 2024-11-16T10:00:25,345 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731751217321.75a5a39ee7bf008864556f27870615b9., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=75a5a39ee7bf008864556f27870615b9, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:00:25,345 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731751217321.75a5a39ee7bf008864556f27870615b9., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 because the exception is null or not the one we care about 2024-11-16T10:00:25,345 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731751217321.75a5a39ee7bf008864556f27870615b9., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 , the old value is region=TestAcidGuarantees,,1731751217321.75a5a39ee7bf008864556f27870615b9., hostname=037c58a19ddb,37267,1731751077844, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=75a5a39ee7bf008864556f27870615b9, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:00:25,345 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731751217321.75a5a39ee7bf008864556f27870615b9., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=75a5a39ee7bf008864556f27870615b9, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:00:25,345 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731751217321.75a5a39ee7bf008864556f27870615b9., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 because the exception is null or not the one we care about 2024-11-16T10:00:25,408 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:25,411 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241116a4d23bce912c47b1accd9d147df295f7_75a5a39ee7bf008864556f27870615b9 to hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241116a4d23bce912c47b1accd9d147df295f7_75a5a39ee7bf008864556f27870615b9 2024-11-16T10:00:25,412 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/75a5a39ee7bf008864556f27870615b9/.tmp/A/7559ef990f3f4cefab9b80a49e667063, store: [table=TestAcidGuarantees family=A region=75a5a39ee7bf008864556f27870615b9] 2024-11-16T10:00:25,413 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/75a5a39ee7bf008864556f27870615b9/.tmp/A/7559ef990f3f4cefab9b80a49e667063 is 175, key is test_row_0/A:col10/1731751224995/Put/seqid=0 2024-11-16T10:00:25,416 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37591 is added to blk_1073742388_1564 (size=39549) 2024-11-16T10:00:25,654 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=75a5a39ee7bf008864556f27870615b9, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T10:00:25,654 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34428 deadline: 1731751285653, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=75a5a39ee7bf008864556f27870615b9, server=037c58a19ddb,37267,1731751077844 2024-11-16T10:00:25,654 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=75a5a39ee7bf008864556f27870615b9, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T10:00:25,654 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731751217321.75a5a39ee7bf008864556f27870615b9., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 , the old value is region=TestAcidGuarantees,,1731751217321.75a5a39ee7bf008864556f27870615b9., hostname=037c58a19ddb,37267,1731751077844, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=75a5a39ee7bf008864556f27870615b9, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:00:25,654 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34454 deadline: 1731751285653, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=75a5a39ee7bf008864556f27870615b9, server=037c58a19ddb,37267,1731751077844 2024-11-16T10:00:25,654 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731751217321.75a5a39ee7bf008864556f27870615b9., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=75a5a39ee7bf008864556f27870615b9, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:00:25,654 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731751217321.75a5a39ee7bf008864556f27870615b9., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 because the exception is null or not the one we care about 2024-11-16T10:00:25,654 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=75a5a39ee7bf008864556f27870615b9, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T10:00:25,655 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731751217321.75a5a39ee7bf008864556f27870615b9., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 , the old value is region=TestAcidGuarantees,,1731751217321.75a5a39ee7bf008864556f27870615b9., hostname=037c58a19ddb,37267,1731751077844, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=75a5a39ee7bf008864556f27870615b9, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:00:25,655 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34430 deadline: 1731751285653, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=75a5a39ee7bf008864556f27870615b9, server=037c58a19ddb,37267,1731751077844 2024-11-16T10:00:25,655 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731751217321.75a5a39ee7bf008864556f27870615b9., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=75a5a39ee7bf008864556f27870615b9, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:00:25,655 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731751217321.75a5a39ee7bf008864556f27870615b9., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 because the exception is null or not the one we care about 2024-11-16T10:00:25,655 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=75a5a39ee7bf008864556f27870615b9, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T10:00:25,655 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34412 deadline: 1731751285653, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=75a5a39ee7bf008864556f27870615b9, server=037c58a19ddb,37267,1731751077844 2024-11-16T10:00:25,655 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731751217321.75a5a39ee7bf008864556f27870615b9., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 , the old value is region=TestAcidGuarantees,,1731751217321.75a5a39ee7bf008864556f27870615b9., hostname=037c58a19ddb,37267,1731751077844, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=75a5a39ee7bf008864556f27870615b9, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:00:25,655 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731751217321.75a5a39ee7bf008864556f27870615b9., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=75a5a39ee7bf008864556f27870615b9, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:00:25,655 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731751217321.75a5a39ee7bf008864556f27870615b9., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 because the exception is null or not the one we care about 2024-11-16T10:00:25,655 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=75a5a39ee7bf008864556f27870615b9, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T10:00:25,655 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731751217321.75a5a39ee7bf008864556f27870615b9., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 , the old value is region=TestAcidGuarantees,,1731751217321.75a5a39ee7bf008864556f27870615b9., hostname=037c58a19ddb,37267,1731751077844, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=75a5a39ee7bf008864556f27870615b9, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:00:25,655 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731751217321.75a5a39ee7bf008864556f27870615b9., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=75a5a39ee7bf008864556f27870615b9, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:00:25,655 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34392 deadline: 1731751285653, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=75a5a39ee7bf008864556f27870615b9, server=037c58a19ddb,37267,1731751077844 2024-11-16T10:00:25,655 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731751217321.75a5a39ee7bf008864556f27870615b9., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 because the exception is null or not the one we care about 2024-11-16T10:00:25,655 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731751217321.75a5a39ee7bf008864556f27870615b9., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 , the old value is region=TestAcidGuarantees,,1731751217321.75a5a39ee7bf008864556f27870615b9., hostname=037c58a19ddb,37267,1731751077844, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=75a5a39ee7bf008864556f27870615b9, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:00:25,655 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731751217321.75a5a39ee7bf008864556f27870615b9., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=75a5a39ee7bf008864556f27870615b9, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:00:25,655 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731751217321.75a5a39ee7bf008864556f27870615b9., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 because the exception is null or not the one we care about 2024-11-16T10:00:25,817 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=95, memsize=29.1 K, hasBloomFilter=true, into tmp file hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/75a5a39ee7bf008864556f27870615b9/.tmp/A/7559ef990f3f4cefab9b80a49e667063 2024-11-16T10:00:25,824 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/75a5a39ee7bf008864556f27870615b9/.tmp/B/34673bb2b5ce43d1825af91265a180ad is 50, key is test_row_0/B:col10/1731751224995/Put/seqid=0 2024-11-16T10:00:25,827 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37591 is added to blk_1073742389_1565 (size=12001) 2024-11-16T10:00:26,164 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=75a5a39ee7bf008864556f27870615b9, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T10:00:26,164 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34428 deadline: 1731751286163, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=75a5a39ee7bf008864556f27870615b9, server=037c58a19ddb,37267,1731751077844 2024-11-16T10:00:26,165 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=75a5a39ee7bf008864556f27870615b9, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T10:00:26,165 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34454 deadline: 1731751286163, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=75a5a39ee7bf008864556f27870615b9, server=037c58a19ddb,37267,1731751077844 2024-11-16T10:00:26,165 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731751217321.75a5a39ee7bf008864556f27870615b9., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 , the old value is region=TestAcidGuarantees,,1731751217321.75a5a39ee7bf008864556f27870615b9., hostname=037c58a19ddb,37267,1731751077844, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=75a5a39ee7bf008864556f27870615b9, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:00:26,165 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731751217321.75a5a39ee7bf008864556f27870615b9., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=75a5a39ee7bf008864556f27870615b9, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:00:26,165 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731751217321.75a5a39ee7bf008864556f27870615b9., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 because the exception is null or not the one we care about 2024-11-16T10:00:26,165 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=75a5a39ee7bf008864556f27870615b9, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T10:00:26,165 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731751217321.75a5a39ee7bf008864556f27870615b9., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 , the old value is region=TestAcidGuarantees,,1731751217321.75a5a39ee7bf008864556f27870615b9., hostname=037c58a19ddb,37267,1731751077844, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=75a5a39ee7bf008864556f27870615b9, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:00:26,165 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34430 deadline: 1731751286163, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=75a5a39ee7bf008864556f27870615b9, server=037c58a19ddb,37267,1731751077844 2024-11-16T10:00:26,165 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731751217321.75a5a39ee7bf008864556f27870615b9., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=75a5a39ee7bf008864556f27870615b9, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:00:26,165 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731751217321.75a5a39ee7bf008864556f27870615b9., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 because the exception is null or not the one we care about 2024-11-16T10:00:26,165 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=75a5a39ee7bf008864556f27870615b9, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T10:00:26,165 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731751217321.75a5a39ee7bf008864556f27870615b9., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 , the old value is region=TestAcidGuarantees,,1731751217321.75a5a39ee7bf008864556f27870615b9., hostname=037c58a19ddb,37267,1731751077844, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=75a5a39ee7bf008864556f27870615b9, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:00:26,165 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34412 deadline: 1731751286163, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=75a5a39ee7bf008864556f27870615b9, server=037c58a19ddb,37267,1731751077844 2024-11-16T10:00:26,165 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731751217321.75a5a39ee7bf008864556f27870615b9., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=75a5a39ee7bf008864556f27870615b9, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:00:26,165 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731751217321.75a5a39ee7bf008864556f27870615b9., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 because the exception is null or not the one we care about 2024-11-16T10:00:26,165 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=75a5a39ee7bf008864556f27870615b9, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T10:00:26,166 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731751217321.75a5a39ee7bf008864556f27870615b9., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 , the old value is region=TestAcidGuarantees,,1731751217321.75a5a39ee7bf008864556f27870615b9., hostname=037c58a19ddb,37267,1731751077844, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=75a5a39ee7bf008864556f27870615b9, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:00:26,166 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731751217321.75a5a39ee7bf008864556f27870615b9., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=75a5a39ee7bf008864556f27870615b9, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:00:26,166 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34392 deadline: 1731751286163, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=75a5a39ee7bf008864556f27870615b9, server=037c58a19ddb,37267,1731751077844 2024-11-16T10:00:26,166 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731751217321.75a5a39ee7bf008864556f27870615b9., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 because the exception is null or not the one we care about 2024-11-16T10:00:26,166 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731751217321.75a5a39ee7bf008864556f27870615b9., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 , the old value is region=TestAcidGuarantees,,1731751217321.75a5a39ee7bf008864556f27870615b9., hostname=037c58a19ddb,37267,1731751077844, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=75a5a39ee7bf008864556f27870615b9, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:00:26,166 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731751217321.75a5a39ee7bf008864556f27870615b9., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=75a5a39ee7bf008864556f27870615b9, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:00:26,166 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731751217321.75a5a39ee7bf008864556f27870615b9., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 because the exception is null or not the one we care about 2024-11-16T10:00:26,227 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=95 (bloomFilter=true), to=hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/75a5a39ee7bf008864556f27870615b9/.tmp/B/34673bb2b5ce43d1825af91265a180ad 2024-11-16T10:00:26,234 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/75a5a39ee7bf008864556f27870615b9/.tmp/C/d29bb71ca61440e28f0cbc943b187152 is 50, key is test_row_0/C:col10/1731751224995/Put/seqid=0 2024-11-16T10:00:26,236 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtil$FsDatasetAsyncDiskServiceFixer(597): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details. 2024-11-16T10:00:26,237 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37591 is added to blk_1073742390_1566 (size=12001) 2024-11-16T10:00:26,237 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=95 (bloomFilter=true), to=hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/75a5a39ee7bf008864556f27870615b9/.tmp/C/d29bb71ca61440e28f0cbc943b187152 2024-11-16T10:00:26,241 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/75a5a39ee7bf008864556f27870615b9/.tmp/A/7559ef990f3f4cefab9b80a49e667063 as hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/75a5a39ee7bf008864556f27870615b9/A/7559ef990f3f4cefab9b80a49e667063 2024-11-16T10:00:26,244 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/75a5a39ee7bf008864556f27870615b9/A/7559ef990f3f4cefab9b80a49e667063, entries=200, sequenceid=95, filesize=38.6 K 2024-11-16T10:00:26,245 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/75a5a39ee7bf008864556f27870615b9/.tmp/B/34673bb2b5ce43d1825af91265a180ad as hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/75a5a39ee7bf008864556f27870615b9/B/34673bb2b5ce43d1825af91265a180ad 2024-11-16T10:00:26,248 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/75a5a39ee7bf008864556f27870615b9/B/34673bb2b5ce43d1825af91265a180ad, entries=150, sequenceid=95, filesize=11.7 K 2024-11-16T10:00:26,249 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/75a5a39ee7bf008864556f27870615b9/.tmp/C/d29bb71ca61440e28f0cbc943b187152 as hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/75a5a39ee7bf008864556f27870615b9/C/d29bb71ca61440e28f0cbc943b187152 2024-11-16T10:00:26,252 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/75a5a39ee7bf008864556f27870615b9/C/d29bb71ca61440e28f0cbc943b187152, entries=150, sequenceid=95, filesize=11.7 K 2024-11-16T10:00:26,255 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~87.22 KB/89310, heapSize ~229.22 KB/234720, currentSize=114.05 KB/116790 for 75a5a39ee7bf008864556f27870615b9 in 1258ms, sequenceid=95, compaction requested=true 2024-11-16T10:00:26,255 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 75a5a39ee7bf008864556f27870615b9: 2024-11-16T10:00:26,255 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 75a5a39ee7bf008864556f27870615b9:A, priority=-2147483648, current under compaction store size is 1 2024-11-16T10:00:26,255 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-16T10:00:26,255 DEBUG [RS:0;037c58a19ddb:37267-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-16T10:00:26,255 DEBUG [RS:0;037c58a19ddb:37267-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-16T10:00:26,255 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 75a5a39ee7bf008864556f27870615b9:B, priority=-2147483648, current under compaction store size is 2 2024-11-16T10:00:26,256 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-16T10:00:26,256 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 75a5a39ee7bf008864556f27870615b9:C, priority=-2147483648, current under compaction store size is 3 2024-11-16T10:00:26,256 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-16T10:00:26,256 DEBUG [RS:0;037c58a19ddb:37267-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 101562 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-16T10:00:26,256 DEBUG [RS:0;037c58a19ddb:37267-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36106 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-16T10:00:26,256 DEBUG [RS:0;037c58a19ddb:37267-shortCompactions-0 {}] regionserver.HStore(1541): 75a5a39ee7bf008864556f27870615b9/A is initiating minor compaction (all files) 2024-11-16T10:00:26,256 DEBUG [RS:0;037c58a19ddb:37267-longCompactions-0 {}] regionserver.HStore(1541): 75a5a39ee7bf008864556f27870615b9/B is initiating minor compaction (all files) 2024-11-16T10:00:26,256 INFO [RS:0;037c58a19ddb:37267-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 75a5a39ee7bf008864556f27870615b9/A in TestAcidGuarantees,,1731751217321.75a5a39ee7bf008864556f27870615b9. 2024-11-16T10:00:26,256 INFO [RS:0;037c58a19ddb:37267-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 75a5a39ee7bf008864556f27870615b9/B in TestAcidGuarantees,,1731751217321.75a5a39ee7bf008864556f27870615b9. 2024-11-16T10:00:26,256 INFO [RS:0;037c58a19ddb:37267-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/75a5a39ee7bf008864556f27870615b9/B/2a05f259717c4e27b74a376fd6d40bf9, hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/75a5a39ee7bf008864556f27870615b9/B/3807f7330657475a8b56368c81de0fe8, hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/75a5a39ee7bf008864556f27870615b9/B/34673bb2b5ce43d1825af91265a180ad] into tmpdir=hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/75a5a39ee7bf008864556f27870615b9/.tmp, totalSize=35.3 K 2024-11-16T10:00:26,256 INFO [RS:0;037c58a19ddb:37267-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/75a5a39ee7bf008864556f27870615b9/A/56efa3c087bc425a838c68ca08f65887, hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/75a5a39ee7bf008864556f27870615b9/A/9712b27b0e4f495e9650ab7a025b0ba1, hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/75a5a39ee7bf008864556f27870615b9/A/7559ef990f3f4cefab9b80a49e667063] into tmpdir=hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/75a5a39ee7bf008864556f27870615b9/.tmp, totalSize=99.2 K 2024-11-16T10:00:26,256 INFO [RS:0;037c58a19ddb:37267-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731751217321.75a5a39ee7bf008864556f27870615b9. 2024-11-16T10:00:26,256 DEBUG [RS:0;037c58a19ddb:37267-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731751217321.75a5a39ee7bf008864556f27870615b9. files: [hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/75a5a39ee7bf008864556f27870615b9/A/56efa3c087bc425a838c68ca08f65887, hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/75a5a39ee7bf008864556f27870615b9/A/9712b27b0e4f495e9650ab7a025b0ba1, hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/75a5a39ee7bf008864556f27870615b9/A/7559ef990f3f4cefab9b80a49e667063] 2024-11-16T10:00:26,257 DEBUG [RS:0;037c58a19ddb:37267-longCompactions-0 {}] compactions.Compactor(225): Compacting 2a05f259717c4e27b74a376fd6d40bf9, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=52, earliestPutTs=1731751222656 2024-11-16T10:00:26,257 DEBUG [RS:0;037c58a19ddb:37267-shortCompactions-0 {}] compactions.Compactor(225): Compacting 56efa3c087bc425a838c68ca08f65887, keycount=150, bloomtype=ROW, size=30.3 K, encoding=NONE, compression=NONE, seqNum=52, earliestPutTs=1731751222656 2024-11-16T10:00:26,257 DEBUG [RS:0;037c58a19ddb:37267-longCompactions-0 {}] compactions.Compactor(225): Compacting 3807f7330657475a8b56368c81de0fe8, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=76, earliestPutTs=1731751222690 2024-11-16T10:00:26,257 DEBUG [RS:0;037c58a19ddb:37267-shortCompactions-0 {}] compactions.Compactor(225): Compacting 9712b27b0e4f495e9650ab7a025b0ba1, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=76, earliestPutTs=1731751222690 2024-11-16T10:00:26,257 DEBUG [RS:0;037c58a19ddb:37267-longCompactions-0 {}] compactions.Compactor(225): Compacting 34673bb2b5ce43d1825af91265a180ad, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=95, earliestPutTs=1731751223859 2024-11-16T10:00:26,257 DEBUG [RS:0;037c58a19ddb:37267-shortCompactions-0 {}] compactions.Compactor(225): Compacting 7559ef990f3f4cefab9b80a49e667063, keycount=200, bloomtype=ROW, size=38.6 K, encoding=NONE, compression=NONE, seqNum=95, earliestPutTs=1731751223853 2024-11-16T10:00:26,262 INFO [RS:0;037c58a19ddb:37267-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=75a5a39ee7bf008864556f27870615b9] 2024-11-16T10:00:26,264 INFO [RS:0;037c58a19ddb:37267-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 75a5a39ee7bf008864556f27870615b9#B#compaction#484 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-16T10:00:26,264 DEBUG [RS:0;037c58a19ddb:37267-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/75a5a39ee7bf008864556f27870615b9/.tmp/B/371f5fd3a9984c5a9022b16ad2d69107 is 50, key is test_row_0/B:col10/1731751224995/Put/seqid=0 2024-11-16T10:00:26,266 DEBUG [RS:0;037c58a19ddb:37267-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e2024111658df797ae7c347aa867527cc12357b15_75a5a39ee7bf008864556f27870615b9 store=[table=TestAcidGuarantees family=A region=75a5a39ee7bf008864556f27870615b9] 2024-11-16T10:00:26,268 DEBUG [RS:0;037c58a19ddb:37267-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e2024111658df797ae7c347aa867527cc12357b15_75a5a39ee7bf008864556f27870615b9, store=[table=TestAcidGuarantees family=A region=75a5a39ee7bf008864556f27870615b9] 2024-11-16T10:00:26,268 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37591 is added to blk_1073742391_1567 (size=12207) 2024-11-16T10:00:26,268 DEBUG [RS:0;037c58a19ddb:37267-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111658df797ae7c347aa867527cc12357b15_75a5a39ee7bf008864556f27870615b9 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=75a5a39ee7bf008864556f27870615b9] 2024-11-16T10:00:26,272 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37591 is added to blk_1073742392_1568 (size=4469) 2024-11-16T10:00:26,273 DEBUG [RS:0;037c58a19ddb:37267-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/75a5a39ee7bf008864556f27870615b9/.tmp/B/371f5fd3a9984c5a9022b16ad2d69107 as hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/75a5a39ee7bf008864556f27870615b9/B/371f5fd3a9984c5a9022b16ad2d69107 2024-11-16T10:00:26,274 INFO [RS:0;037c58a19ddb:37267-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 75a5a39ee7bf008864556f27870615b9#A#compaction#485 average throughput is 2.04 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-16T10:00:26,274 DEBUG [RS:0;037c58a19ddb:37267-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/75a5a39ee7bf008864556f27870615b9/.tmp/A/c45414a6fed343d78f84a9867a4a9bfc is 175, key is test_row_0/A:col10/1731751224995/Put/seqid=0 2024-11-16T10:00:26,278 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37591 is added to blk_1073742393_1569 (size=31161) 2024-11-16T10:00:26,279 INFO [RS:0;037c58a19ddb:37267-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 75a5a39ee7bf008864556f27870615b9/B of 75a5a39ee7bf008864556f27870615b9 into 371f5fd3a9984c5a9022b16ad2d69107(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-16T10:00:26,279 DEBUG [RS:0;037c58a19ddb:37267-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 75a5a39ee7bf008864556f27870615b9: 2024-11-16T10:00:26,279 INFO [RS:0;037c58a19ddb:37267-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731751217321.75a5a39ee7bf008864556f27870615b9., storeName=75a5a39ee7bf008864556f27870615b9/B, priority=13, startTime=1731751226255; duration=0sec 2024-11-16T10:00:26,279 DEBUG [RS:0;037c58a19ddb:37267-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-16T10:00:26,279 DEBUG [RS:0;037c58a19ddb:37267-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 75a5a39ee7bf008864556f27870615b9:B 2024-11-16T10:00:26,279 DEBUG [RS:0;037c58a19ddb:37267-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-16T10:00:26,280 DEBUG [RS:0;037c58a19ddb:37267-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36106 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-16T10:00:26,280 DEBUG [RS:0;037c58a19ddb:37267-longCompactions-0 {}] regionserver.HStore(1541): 75a5a39ee7bf008864556f27870615b9/C is initiating minor compaction (all files) 2024-11-16T10:00:26,280 INFO [RS:0;037c58a19ddb:37267-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 75a5a39ee7bf008864556f27870615b9/C in TestAcidGuarantees,,1731751217321.75a5a39ee7bf008864556f27870615b9. 2024-11-16T10:00:26,280 INFO [RS:0;037c58a19ddb:37267-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/75a5a39ee7bf008864556f27870615b9/C/660a6b8d351a4709a39d1668e126c1e3, hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/75a5a39ee7bf008864556f27870615b9/C/4634faf075a148e0aa55658f0b57d51f, hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/75a5a39ee7bf008864556f27870615b9/C/d29bb71ca61440e28f0cbc943b187152] into tmpdir=hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/75a5a39ee7bf008864556f27870615b9/.tmp, totalSize=35.3 K 2024-11-16T10:00:26,280 DEBUG [RS:0;037c58a19ddb:37267-longCompactions-0 {}] compactions.Compactor(225): Compacting 660a6b8d351a4709a39d1668e126c1e3, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=52, earliestPutTs=1731751222656 2024-11-16T10:00:26,281 DEBUG [RS:0;037c58a19ddb:37267-longCompactions-0 {}] compactions.Compactor(225): Compacting 4634faf075a148e0aa55658f0b57d51f, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=76, earliestPutTs=1731751222690 2024-11-16T10:00:26,281 DEBUG [RS:0;037c58a19ddb:37267-longCompactions-0 {}] compactions.Compactor(225): Compacting d29bb71ca61440e28f0cbc943b187152, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=95, earliestPutTs=1731751223859 2024-11-16T10:00:26,286 INFO [RS:0;037c58a19ddb:37267-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 75a5a39ee7bf008864556f27870615b9#C#compaction#486 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-16T10:00:26,287 DEBUG [RS:0;037c58a19ddb:37267-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/75a5a39ee7bf008864556f27870615b9/.tmp/C/c36f6d576030452db8db9b4cf83d536e is 50, key is test_row_0/C:col10/1731751224995/Put/seqid=0 2024-11-16T10:00:26,290 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37591 is added to blk_1073742394_1570 (size=12207) 2024-11-16T10:00:26,683 DEBUG [RS:0;037c58a19ddb:37267-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/75a5a39ee7bf008864556f27870615b9/.tmp/A/c45414a6fed343d78f84a9867a4a9bfc as hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/75a5a39ee7bf008864556f27870615b9/A/c45414a6fed343d78f84a9867a4a9bfc 2024-11-16T10:00:26,687 INFO [RS:0;037c58a19ddb:37267-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 75a5a39ee7bf008864556f27870615b9/A of 75a5a39ee7bf008864556f27870615b9 into c45414a6fed343d78f84a9867a4a9bfc(size=30.4 K), total size for store is 30.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-16T10:00:26,687 DEBUG [RS:0;037c58a19ddb:37267-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 75a5a39ee7bf008864556f27870615b9: 2024-11-16T10:00:26,687 INFO [RS:0;037c58a19ddb:37267-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731751217321.75a5a39ee7bf008864556f27870615b9., storeName=75a5a39ee7bf008864556f27870615b9/A, priority=13, startTime=1731751226255; duration=0sec 2024-11-16T10:00:26,687 DEBUG [RS:0;037c58a19ddb:37267-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-16T10:00:26,687 DEBUG [RS:0;037c58a19ddb:37267-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 75a5a39ee7bf008864556f27870615b9:A 2024-11-16T10:00:26,694 DEBUG [RS:0;037c58a19ddb:37267-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/75a5a39ee7bf008864556f27870615b9/.tmp/C/c36f6d576030452db8db9b4cf83d536e as hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/75a5a39ee7bf008864556f27870615b9/C/c36f6d576030452db8db9b4cf83d536e 2024-11-16T10:00:26,698 INFO [RS:0;037c58a19ddb:37267-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 75a5a39ee7bf008864556f27870615b9/C of 75a5a39ee7bf008864556f27870615b9 into c36f6d576030452db8db9b4cf83d536e(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-16T10:00:26,698 DEBUG [RS:0;037c58a19ddb:37267-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 75a5a39ee7bf008864556f27870615b9: 2024-11-16T10:00:26,698 INFO [RS:0;037c58a19ddb:37267-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731751217321.75a5a39ee7bf008864556f27870615b9., storeName=75a5a39ee7bf008864556f27870615b9/C, priority=13, startTime=1731751226256; duration=0sec 2024-11-16T10:00:26,698 DEBUG [RS:0;037c58a19ddb:37267-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-16T10:00:26,698 DEBUG [RS:0;037c58a19ddb:37267-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 75a5a39ee7bf008864556f27870615b9:C 2024-11-16T10:00:26,743 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37877 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=152 2024-11-16T10:00:26,743 INFO [RPCClient-NioEventLoopGroup-4-12 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-16T10:00:26,743 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37877 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-16T10:00:26,744 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37877 {}] procedure2.ProcedureExecutor(1139): Stored pid=154, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=154, table=TestAcidGuarantees 2024-11-16T10:00:26,745 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37877 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=154 2024-11-16T10:00:26,745 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=154, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=154, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-16T10:00:26,746 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=154, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=154, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-16T10:00:26,746 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=155, ppid=154, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-16T10:00:26,852 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37877 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=154 2024-11-16T10:00:26,898 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=37267 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=155 2024-11-16T10:00:26,898 DEBUG [RS_FLUSH_OPERATIONS-regionserver/037c58a19ddb:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731751217321.75a5a39ee7bf008864556f27870615b9. 2024-11-16T10:00:26,898 INFO [RS_FLUSH_OPERATIONS-regionserver/037c58a19ddb:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HRegion(2902): Flushing 75a5a39ee7bf008864556f27870615b9 3/3 column families, dataSize=114.05 KB heapSize=299.58 KB 2024-11-16T10:00:26,905 DEBUG [RS_FLUSH_OPERATIONS-regionserver/037c58a19ddb:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241116b04b93190a8646dc82267e45f8168d2f_75a5a39ee7bf008864556f27870615b9 is 50, key is test_row_0/A:col10/1731751225022/Put/seqid=0 2024-11-16T10:00:26,908 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37591 is added to blk_1073742395_1571 (size=12154) 2024-11-16T10:00:26,909 DEBUG [RS_FLUSH_OPERATIONS-regionserver/037c58a19ddb:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:26,913 INFO [RS_FLUSH_OPERATIONS-regionserver/037c58a19ddb:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241116b04b93190a8646dc82267e45f8168d2f_75a5a39ee7bf008864556f27870615b9 to hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241116b04b93190a8646dc82267e45f8168d2f_75a5a39ee7bf008864556f27870615b9 2024-11-16T10:00:26,914 DEBUG [RS_FLUSH_OPERATIONS-regionserver/037c58a19ddb:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/75a5a39ee7bf008864556f27870615b9/.tmp/A/a229e16c29e64ca982c262db7c64505c, store: [table=TestAcidGuarantees family=A region=75a5a39ee7bf008864556f27870615b9] 2024-11-16T10:00:26,915 DEBUG [RS_FLUSH_OPERATIONS-regionserver/037c58a19ddb:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/75a5a39ee7bf008864556f27870615b9/.tmp/A/a229e16c29e64ca982c262db7c64505c is 175, key is test_row_0/A:col10/1731751225022/Put/seqid=0 2024-11-16T10:00:26,918 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37591 is added to blk_1073742396_1572 (size=30955) 2024-11-16T10:00:26,918 INFO [RS_FLUSH_OPERATIONS-regionserver/037c58a19ddb:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=118, memsize=38.0 K, hasBloomFilter=true, into tmp file hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/75a5a39ee7bf008864556f27870615b9/.tmp/A/a229e16c29e64ca982c262db7c64505c 2024-11-16T10:00:26,925 DEBUG [RS_FLUSH_OPERATIONS-regionserver/037c58a19ddb:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/75a5a39ee7bf008864556f27870615b9/.tmp/B/8c92dd9bd0d34f40b1dab0876669e75e is 50, key is test_row_0/B:col10/1731751225022/Put/seqid=0 2024-11-16T10:00:26,927 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37591 is added to blk_1073742397_1573 (size=12001) 2024-11-16T10:00:26,928 INFO [RS_FLUSH_OPERATIONS-regionserver/037c58a19ddb:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=118 (bloomFilter=true), to=hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/75a5a39ee7bf008864556f27870615b9/.tmp/B/8c92dd9bd0d34f40b1dab0876669e75e 2024-11-16T10:00:26,934 DEBUG [RS_FLUSH_OPERATIONS-regionserver/037c58a19ddb:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/75a5a39ee7bf008864556f27870615b9/.tmp/C/6b0168243cd5427aa60eaa61687c5d65 is 50, key is test_row_0/C:col10/1731751225022/Put/seqid=0 2024-11-16T10:00:26,937 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37591 is added to blk_1073742398_1574 (size=12001) 2024-11-16T10:00:26,938 INFO [RS_FLUSH_OPERATIONS-regionserver/037c58a19ddb:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=118 (bloomFilter=true), to=hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/75a5a39ee7bf008864556f27870615b9/.tmp/C/6b0168243cd5427aa60eaa61687c5d65 2024-11-16T10:00:26,942 DEBUG [RS_FLUSH_OPERATIONS-regionserver/037c58a19ddb:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/75a5a39ee7bf008864556f27870615b9/.tmp/A/a229e16c29e64ca982c262db7c64505c as hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/75a5a39ee7bf008864556f27870615b9/A/a229e16c29e64ca982c262db7c64505c 2024-11-16T10:00:26,945 INFO [RS_FLUSH_OPERATIONS-regionserver/037c58a19ddb:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/75a5a39ee7bf008864556f27870615b9/A/a229e16c29e64ca982c262db7c64505c, entries=150, sequenceid=118, filesize=30.2 K 2024-11-16T10:00:26,946 DEBUG [RS_FLUSH_OPERATIONS-regionserver/037c58a19ddb:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/75a5a39ee7bf008864556f27870615b9/.tmp/B/8c92dd9bd0d34f40b1dab0876669e75e as hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/75a5a39ee7bf008864556f27870615b9/B/8c92dd9bd0d34f40b1dab0876669e75e 2024-11-16T10:00:26,946 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:26,946 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:26,947 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:26,949 INFO [RS_FLUSH_OPERATIONS-regionserver/037c58a19ddb:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/75a5a39ee7bf008864556f27870615b9/B/8c92dd9bd0d34f40b1dab0876669e75e, entries=150, sequenceid=118, filesize=11.7 K 2024-11-16T10:00:26,950 DEBUG [RS_FLUSH_OPERATIONS-regionserver/037c58a19ddb:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/75a5a39ee7bf008864556f27870615b9/.tmp/C/6b0168243cd5427aa60eaa61687c5d65 as hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/75a5a39ee7bf008864556f27870615b9/C/6b0168243cd5427aa60eaa61687c5d65 2024-11-16T10:00:26,950 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:26,950 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:26,950 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:26,950 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:26,950 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:26,950 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:26,951 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:26,951 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:26,951 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:26,951 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:26,951 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:26,951 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:26,952 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:26,952 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:26,952 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:26,952 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:26,952 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:26,953 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:26,953 INFO [RS_FLUSH_OPERATIONS-regionserver/037c58a19ddb:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/75a5a39ee7bf008864556f27870615b9/C/6b0168243cd5427aa60eaa61687c5d65, entries=150, sequenceid=118, filesize=11.7 K 2024-11-16T10:00:26,953 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:26,953 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:26,953 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:26,953 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:26,954 INFO [RS_FLUSH_OPERATIONS-regionserver/037c58a19ddb:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HRegion(3140): Finished flush of dataSize ~114.05 KB/116790, heapSize ~299.53 KB/306720, currentSize=0 B/0 for 75a5a39ee7bf008864556f27870615b9 in 55ms, sequenceid=118, compaction requested=false 2024-11-16T10:00:26,954 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:26,954 DEBUG [RS_FLUSH_OPERATIONS-regionserver/037c58a19ddb:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HRegion(2603): Flush status journal for 75a5a39ee7bf008864556f27870615b9: 2024-11-16T10:00:26,954 DEBUG [RS_FLUSH_OPERATIONS-regionserver/037c58a19ddb:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731751217321.75a5a39ee7bf008864556f27870615b9. 2024-11-16T10:00:26,954 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:26,954 DEBUG [RS_FLUSH_OPERATIONS-regionserver/037c58a19ddb:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=155 2024-11-16T10:00:26,954 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:26,954 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:26,954 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:26,954 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37877 {}] master.HMaster(4169): Remote procedure done, pid=155 2024-11-16T10:00:26,955 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:26,955 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:26,955 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:26,955 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:26,955 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:26,955 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:26,956 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:26,956 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:26,956 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:26,956 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:26,957 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:26,957 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:26,957 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:26,957 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:26,957 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:26,958 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=155, resume processing ppid=154 2024-11-16T10:00:26,958 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=155, ppid=154, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 209 msec 2024-11-16T10:00:26,958 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:26,958 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:26,958 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:26,958 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:26,958 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:26,959 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:26,959 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:26,959 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:26,959 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:26,959 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:26,959 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:26,960 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:26,960 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:26,960 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=154, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=154, table=TestAcidGuarantees in 216 msec 2024-11-16T10:00:26,960 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:26,960 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:26,960 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:26,961 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:26,961 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:26,961 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:26,961 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:26,961 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:26,961 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:26,962 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:26,962 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:26,962 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:26,962 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:26,962 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:26,962 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:26,963 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:26,963 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:26,963 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:26,963 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:26,964 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:26,964 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:26,964 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:26,965 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:26,965 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:26,965 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:26,966 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:26,966 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:26,966 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:26,966 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:26,967 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:26,967 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:26,967 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:26,967 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:26,967 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:26,968 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:26,968 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:26,968 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:26,968 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:26,968 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:26,968 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:26,969 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:26,969 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:26,969 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:26,969 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:26,969 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:26,969 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:26,970 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:26,970 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:26,970 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:26,970 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:26,970 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:26,970 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:26,971 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:26,971 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:26,971 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:26,971 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:26,971 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:26,971 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:26,972 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:26,972 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:26,972 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:26,972 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:26,972 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:26,973 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:26,973 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:26,973 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:26,973 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:26,973 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:26,973 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:26,974 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:26,974 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:26,974 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:26,974 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:26,974 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:26,975 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:26,975 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:26,975 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:26,975 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:26,975 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:26,975 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:26,976 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:26,976 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:26,976 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:26,976 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:26,976 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:26,977 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:26,977 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:26,977 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:26,977 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:26,978 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:26,978 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:26,978 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:26,979 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:26,979 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:26,979 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:26,979 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:26,980 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:26,980 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:26,980 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:26,980 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:26,980 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:26,981 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:26,981 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:26,981 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:26,981 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:26,981 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:26,981 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:26,981 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:26,982 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:26,982 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:26,982 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:26,982 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:26,982 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:26,983 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:26,983 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:26,983 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:26,983 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:26,983 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:26,983 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:26,984 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:26,984 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:26,984 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:26,984 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:26,984 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:26,985 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:26,985 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:26,985 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:26,985 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:26,985 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:26,985 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:26,986 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:26,986 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:26,986 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:26,986 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:26,986 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:26,986 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:26,987 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:26,987 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:26,987 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:26,987 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:26,988 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:26,988 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:26,988 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:26,988 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:26,988 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:26,988 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:26,989 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:26,989 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:26,989 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:26,989 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:26,989 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:26,989 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:26,990 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:26,990 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:26,990 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:26,990 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:26,990 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:26,990 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:26,991 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:26,991 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:26,991 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:26,991 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:26,991 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:26,991 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:26,992 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:26,992 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:26,992 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:26,992 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:26,992 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:26,992 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:26,993 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:26,993 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:26,993 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:26,993 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:26,993 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:26,994 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:26,994 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:26,995 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:26,995 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:26,995 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:26,995 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:26,995 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:26,996 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:26,996 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:26,996 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:26,996 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:26,996 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:26,997 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:26,997 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:26,997 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:26,997 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:26,997 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:26,997 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:26,998 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:26,998 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:26,998 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:26,998 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:26,998 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:26,998 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:26,999 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:26,999 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:26,999 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:26,999 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:26,999 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:26,999 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:27,000 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:27,000 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:27,000 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:27,000 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:27,000 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:27,001 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:27,001 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:27,001 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:27,001 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:27,001 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:27,001 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:27,002 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:27,002 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:27,002 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:27,002 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:27,002 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:27,002 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:27,003 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:27,003 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:27,003 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:27,003 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:27,003 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:27,004 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:27,004 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:27,004 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:27,004 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:27,004 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:27,005 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:27,005 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:27,005 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:27,005 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:27,005 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:27,006 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:27,006 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:27,006 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:27,006 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:27,006 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:27,006 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:27,007 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:27,007 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:27,007 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:27,007 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:27,007 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:27,007 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:27,008 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:27,008 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:27,008 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:27,008 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:27,008 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:27,009 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:27,009 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:27,009 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:27,009 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:27,009 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:27,009 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:27,010 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:27,010 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:27,010 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:27,010 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:27,010 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:27,010 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:27,011 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:27,011 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:27,011 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:27,011 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:27,011 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:27,012 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:27,012 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:27,012 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:27,012 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:27,012 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:27,013 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:27,013 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:27,013 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:27,013 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:27,013 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:27,013 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:27,014 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:27,014 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:27,014 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:27,014 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:27,014 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:27,014 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:27,015 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:27,015 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:27,015 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:27,015 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:27,015 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:27,015 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:27,016 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:27,016 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:27,016 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:27,016 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:27,016 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:27,016 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:27,017 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:27,017 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:27,017 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:27,017 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:27,017 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:27,017 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:27,018 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:27,018 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:27,018 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:27,018 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:27,018 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:27,018 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:27,019 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:27,019 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:27,019 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:27,019 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:27,019 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:27,019 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:27,020 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:27,020 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:27,020 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:27,020 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:27,020 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:27,021 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:27,021 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:27,022 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:27,022 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:27,022 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:27,022 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:27,023 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:27,023 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:27,023 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:27,023 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:27,023 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:27,023 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:27,024 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:27,024 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:27,024 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:27,024 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:27,024 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:27,024 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:27,025 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:27,025 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:27,025 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:27,025 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:27,025 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:27,025 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:27,026 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:27,026 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:27,026 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:27,026 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:27,026 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:27,026 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:27,027 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:27,027 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:27,027 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:27,027 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:27,027 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:27,028 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:27,028 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:27,028 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:27,028 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:27,028 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:27,028 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:27,029 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:27,029 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:27,029 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:27,029 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:27,029 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:27,029 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:27,030 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:27,030 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:27,030 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:27,030 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:27,030 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:27,030 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:27,031 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:27,031 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:27,031 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:27,031 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:27,031 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:27,031 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:27,032 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:27,032 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:27,032 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:27,032 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:27,033 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:27,033 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:27,033 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:27,033 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:27,033 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:27,034 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:27,034 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:27,034 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:27,034 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:27,034 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:27,034 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:27,035 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:27,035 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:27,035 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:27,035 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:27,035 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:27,035 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:27,036 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:27,036 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:27,036 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:27,036 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:27,036 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:27,037 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:27,037 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:27,037 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:27,038 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:27,038 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:27,038 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:27,038 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:27,039 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:27,039 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:27,039 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:27,039 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:27,039 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:27,039 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:27,040 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:27,040 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:27,040 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:27,040 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:27,040 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:27,041 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:27,041 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:27,041 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:27,041 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:27,041 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:27,042 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:27,042 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:27,042 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:27,042 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:27,042 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:27,043 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:27,043 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:27,043 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:27,043 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:27,043 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:27,043 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:27,044 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:27,044 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:27,044 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:27,044 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:27,044 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:27,045 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:27,045 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:27,045 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:27,045 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:27,045 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:27,045 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:27,046 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:27,046 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:27,046 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:27,046 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:27,047 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:27,047 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:27,047 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:27,047 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:27,047 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:27,047 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:27,048 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:27,048 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:27,048 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:27,048 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:27,049 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:27,049 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:27,049 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:27,049 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:27,049 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:27,049 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:27,050 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:27,050 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:27,050 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:27,050 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:27,051 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:27,051 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:27,052 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:27,052 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:27,052 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:27,052 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:27,052 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:27,053 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:27,053 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:27,053 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:27,053 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:27,053 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:27,053 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:27,054 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:27,054 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:27,054 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:27,054 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:27,054 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:27,055 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:27,055 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:27,055 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:27,055 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:27,055 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:27,055 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:27,056 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:27,056 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:27,056 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:27,056 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:27,057 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:27,057 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:27,057 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:27,057 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:27,057 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:27,058 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:27,058 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:27,058 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:27,058 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:27,058 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:27,058 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:27,059 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:27,059 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:27,059 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:27,059 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:27,059 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:27,059 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:27,060 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:27,060 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:27,060 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:27,060 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:27,061 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:27,061 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:27,061 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:27,061 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:27,062 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:27,062 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:27,062 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:27,062 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:27,063 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:27,063 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:27,063 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:27,063 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37877 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=154 2024-11-16T10:00:27,063 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:27,063 INFO [RPCClient-NioEventLoopGroup-4-12 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-16T10:00:27,063 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:27,064 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:27,064 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:27,064 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37877 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-16T10:00:27,064 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:27,064 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:27,065 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:27,065 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:27,065 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:27,065 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37877 {}] procedure2.ProcedureExecutor(1139): Stored pid=156, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=156, table=TestAcidGuarantees 2024-11-16T10:00:27,065 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:27,066 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:27,066 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37877 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=156 2024-11-16T10:00:27,066 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:27,066 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=156, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=156, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-16T10:00:27,066 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:27,066 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:27,066 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:27,067 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=156, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=156, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-16T10:00:27,067 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:27,067 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=157, ppid=156, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-16T10:00:27,067 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:27,067 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:27,067 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:27,068 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:27,068 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:27,068 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:27,068 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:27,069 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:27,069 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:27,069 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:27,069 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:27,069 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:27,069 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:27,070 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:27,070 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:27,070 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:27,070 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:27,071 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:27,071 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:27,071 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:27,071 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:27,071 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:27,071 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:27,072 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:27,072 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:27,072 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:27,072 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:27,072 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:27,073 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:27,073 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:27,073 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:27,073 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:27,073 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:27,074 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:27,074 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:27,074 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:27,074 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:27,075 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:27,075 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:27,075 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:27,075 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:27,075 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:27,075 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:27,076 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:27,076 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:27,076 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:27,076 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:27,076 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:27,076 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:27,077 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:27,077 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:27,077 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:27,077 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:27,077 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:27,078 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:27,078 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:27,078 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:27,078 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:27,078 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:27,078 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:27,078 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:27,079 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:27,079 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:27,079 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:27,079 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:27,079 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:27,079 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:27,080 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:27,080 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:27,080 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:27,081 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:27,081 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:27,081 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:27,081 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:27,082 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:27,082 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:27,082 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:27,082 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:27,082 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:27,083 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:27,083 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:27,083 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:27,083 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:27,083 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:27,083 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:27,084 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:27,084 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:27,084 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:27,084 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:27,084 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:27,085 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:27,085 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:27,085 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:27,085 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:27,086 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:27,086 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:27,086 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:27,086 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:27,086 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:27,087 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:27,087 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:27,087 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:27,087 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:27,087 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:27,088 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:27,088 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:27,088 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:27,088 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:27,088 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:27,088 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:27,089 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:27,089 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:27,089 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:27,090 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:27,090 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:27,090 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:27,090 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:27,090 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:27,091 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:27,091 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:27,091 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:27,091 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:27,091 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:27,092 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:27,092 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:27,092 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:27,092 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:27,092 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:27,093 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:27,093 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:27,093 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:27,093 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:27,093 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:27,093 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:27,094 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:27,094 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:27,094 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:27,094 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:27,094 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:27,095 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:27,095 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:27,095 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:27,095 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:27,095 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:27,095 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:27,095 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:27,096 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:27,096 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:27,096 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:27,096 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:27,096 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:27,097 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:27,097 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:27,097 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:27,097 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:27,097 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:27,097 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:27,098 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:27,098 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:27,098 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:27,098 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:27,098 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:27,099 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:27,099 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:27,099 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:27,099 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:27,099 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:27,100 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:27,100 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:27,100 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:27,100 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:27,100 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:27,101 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:27,101 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:27,101 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:27,101 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:27,101 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:27,101 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:27,102 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:27,102 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:27,102 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:27,102 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:27,102 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:27,103 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:27,103 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:27,103 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:27,103 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:27,103 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:27,104 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:27,104 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:27,104 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:27,104 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:27,104 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:27,104 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:27,105 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:27,105 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:27,105 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:27,105 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:27,106 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:27,106 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:27,106 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:27,106 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:27,106 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:27,106 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:27,107 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:27,107 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:27,107 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:27,107 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:27,108 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:27,108 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:27,108 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:27,108 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:27,108 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:27,109 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:27,109 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:27,109 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:27,109 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:27,109 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:27,110 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:27,110 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:27,110 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:27,110 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:27,111 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:27,111 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:27,111 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:27,111 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:27,111 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:27,111 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:27,112 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:27,112 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:27,112 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:27,112 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:27,112 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:27,113 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:27,113 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:27,113 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:27,113 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:27,113 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:27,114 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:27,114 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:27,114 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:27,114 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:27,114 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:27,115 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:27,115 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:27,115 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:27,115 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:27,115 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:27,115 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:27,116 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:27,116 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:27,116 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:27,116 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:27,117 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:27,117 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:27,117 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:27,117 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:27,118 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:27,118 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:27,118 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:27,118 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:27,119 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:27,119 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:27,119 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:27,119 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:27,120 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:27,120 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:27,120 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:27,120 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:27,120 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:27,121 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:27,121 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:27,121 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:27,121 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:27,121 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:27,122 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:27,122 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:27,122 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:27,122 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:27,122 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:27,123 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:27,123 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:27,123 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:27,123 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:27,123 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:27,123 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:27,124 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:27,124 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:27,124 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:27,124 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:27,124 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:27,125 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:27,125 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:27,125 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:27,125 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:27,125 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:27,125 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:27,126 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:27,126 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:27,126 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:27,126 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:27,126 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:27,127 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:27,127 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:27,127 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:27,127 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:27,127 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:27,127 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:27,128 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:27,128 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:27,128 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:27,128 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:27,128 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:27,129 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:27,129 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:27,129 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:27,129 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:27,129 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:27,129 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:27,129 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:27,130 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:27,130 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:27,130 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:27,130 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:27,130 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:27,131 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:27,131 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:27,131 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:27,131 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:27,131 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:27,131 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:27,132 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:27,132 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:27,132 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:27,132 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:27,132 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:27,133 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:27,133 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:27,133 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:27,133 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:27,133 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:27,133 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:27,134 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:27,134 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:27,134 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:27,134 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:27,134 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:27,135 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:27,135 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:27,135 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:27,135 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:27,135 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:27,135 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:27,136 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:27,136 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:27,136 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:27,136 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:27,136 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:27,137 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:27,137 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:27,137 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:27,137 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:27,138 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:27,138 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:27,138 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:27,138 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:27,139 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:27,139 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:27,139 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:27,139 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:27,139 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:27,139 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:27,140 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:27,140 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:27,140 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:27,140 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:27,141 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:27,141 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:27,141 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:27,141 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:27,141 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:27,142 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:27,142 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:27,142 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:27,142 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:27,142 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:27,142 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:27,143 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:27,143 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:27,143 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:27,143 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:27,143 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:27,144 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:27,144 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:27,144 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:27,144 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:27,145 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:27,145 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:27,145 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:27,145 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:27,145 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:27,146 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:27,146 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:27,146 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:27,146 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:27,146 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:27,146 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:27,147 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:27,147 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:27,147 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:27,147 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:27,148 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:27,148 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:27,148 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:27,148 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:27,148 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:27,148 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:27,149 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:27,149 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:27,149 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:27,149 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:27,150 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:27,150 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:27,150 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:27,151 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:27,151 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:27,151 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:27,151 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:27,151 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:27,152 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:27,152 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:27,152 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:27,152 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:27,153 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:27,153 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:27,153 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:27,153 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:27,153 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:27,154 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:27,154 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:27,154 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:27,154 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:27,154 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:27,154 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:27,155 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:27,155 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:27,155 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:27,156 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:27,156 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:27,156 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:27,156 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:27,156 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:27,157 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:27,157 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:27,157 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:27,157 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:27,157 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:27,158 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:27,158 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:27,158 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:27,158 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:27,158 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:27,159 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:27,159 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:27,159 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:27,159 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:27,159 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:27,160 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:27,160 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:27,160 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:27,160 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:27,160 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:27,161 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:27,161 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:27,161 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:27,161 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:27,162 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:27,162 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:27,162 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:27,162 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:27,162 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:27,162 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:27,163 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:27,163 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:27,163 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:27,163 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:27,163 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:27,164 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:27,164 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:27,164 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:27,164 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:27,165 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:27,165 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:27,165 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:27,165 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:27,166 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:27,166 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:27,166 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:27,166 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:27,166 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:27,167 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:27,167 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:27,167 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:27,167 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:27,167 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:27,167 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:27,168 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:27,168 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:27,168 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:27,169 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:27,169 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:27,169 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:27,169 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:27,170 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:27,170 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:27,170 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:27,171 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:27,171 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:27,171 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:27,171 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:27,171 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:27,172 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:27,172 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:27,172 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:27,172 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:27,173 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:27,173 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:27,174 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:27,174 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:27,174 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:27,174 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37877 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=156 2024-11-16T10:00:27,174 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:27,175 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:27,175 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:27,175 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:27,175 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:27,175 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:27,176 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:27,176 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:27,176 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:27,176 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:27,177 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:27,177 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:27,177 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:27,178 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:27,178 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:27,178 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:27,178 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:27,179 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:27,179 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:27,179 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:27,179 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:27,180 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:27,180 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:27,180 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:27,181 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:27,181 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:27,181 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:27,181 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:27,181 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:27,182 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:27,182 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:27,182 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:27,182 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:27,183 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:27,183 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:27,183 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:27,183 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:27,183 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:27,184 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:27,184 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:27,184 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:27,185 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:27,185 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:27,185 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:27,185 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:27,186 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:27,186 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:27,186 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:27,186 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:27,186 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:27,187 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:27,187 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:27,187 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:27,187 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:27,187 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:27,188 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:27,188 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:27,188 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:27,188 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:27,189 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:27,189 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:27,189 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:27,189 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:27,189 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:27,189 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:27,190 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:27,190 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:27,190 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:27,190 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:27,191 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:27,191 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:27,191 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:27,191 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:27,191 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:27,191 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:27,192 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:27,192 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:27,192 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:27,192 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:27,193 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:27,193 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:27,193 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:27,193 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:27,193 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:27,193 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:27,194 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:27,194 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:27,194 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:27,194 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:27,195 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:27,195 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:27,195 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:27,195 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:27,196 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:27,196 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:27,196 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:27,196 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:27,196 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:27,196 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:27,197 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:27,197 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:27,197 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:27,198 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:27,198 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:27,198 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:27,198 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:27,198 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:27,198 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:27,199 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:27,199 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:27,199 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:27,199 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:27,200 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:27,200 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:27,200 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:27,200 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:27,200 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:27,200 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:27,201 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:27,201 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:27,201 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:27,202 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:27,202 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:27,202 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:27,202 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:27,202 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:27,203 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:27,203 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:27,203 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:27,203 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:27,203 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:27,203 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:27,204 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:27,204 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:27,204 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:27,204 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:27,205 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:27,205 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:27,205 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:27,206 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:27,206 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:27,207 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:27,207 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:27,207 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:27,208 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:27,208 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:27,208 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:27,209 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:27,209 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:27,210 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:27,210 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:27,210 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:27,211 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:27,211 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:27,211 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:27,212 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:27,213 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:27,213 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:27,214 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:27,215 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:27,216 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:27,217 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] regionserver.HRegion(8855): Flush requested on 75a5a39ee7bf008864556f27870615b9 2024-11-16T10:00:27,217 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-16T10:00:27,218 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 75a5a39ee7bf008864556f27870615b9 3/3 column families, dataSize=53.67 KB heapSize=141.38 KB 2024-11-16T10:00:27,219 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=37267 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=157 2024-11-16T10:00:27,219 DEBUG [RS_FLUSH_OPERATIONS-regionserver/037c58a19ddb:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731751217321.75a5a39ee7bf008864556f27870615b9. 2024-11-16T10:00:27,219 DEBUG [RS_FLUSH_OPERATIONS-regionserver/037c58a19ddb:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731751217321.75a5a39ee7bf008864556f27870615b9. as already flushing 2024-11-16T10:00:27,219 DEBUG [RS_FLUSH_OPERATIONS-regionserver/037c58a19ddb:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731751217321.75a5a39ee7bf008864556f27870615b9. 2024-11-16T10:00:27,219 ERROR [RS_FLUSH_OPERATIONS-regionserver/037c58a19ddb:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] handler.RSProcedureHandler(58): pid=157 java.io.IOException: Unable to complete flush {ENCODED => 75a5a39ee7bf008864556f27870615b9, NAME => 'TestAcidGuarantees,,1731751217321.75a5a39ee7bf008864556f27870615b9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T10:00:27,220 DEBUG [RS_FLUSH_OPERATIONS-regionserver/037c58a19ddb:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=157 java.io.IOException: Unable to complete flush {ENCODED => 75a5a39ee7bf008864556f27870615b9, NAME => 'TestAcidGuarantees,,1731751217321.75a5a39ee7bf008864556f27870615b9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T10:00:27,221 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37877 {}] master.HMaster(4177): Remote procedure failed, pid=157 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 75a5a39ee7bf008864556f27870615b9, NAME => 'TestAcidGuarantees,,1731751217321.75a5a39ee7bf008864556f27870615b9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 75a5a39ee7bf008864556f27870615b9, NAME => 'TestAcidGuarantees,,1731751217321.75a5a39ee7bf008864556f27870615b9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T10:00:27,227 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411164c7cb85849044effa54b580940139835_75a5a39ee7bf008864556f27870615b9 is 50, key is test_row_0/A:col10/1731751227211/Put/seqid=0 2024-11-16T10:00:27,247 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37591 is added to blk_1073742399_1575 (size=17184) 2024-11-16T10:00:27,249 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:27,253 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411164c7cb85849044effa54b580940139835_75a5a39ee7bf008864556f27870615b9 to hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411164c7cb85849044effa54b580940139835_75a5a39ee7bf008864556f27870615b9 2024-11-16T10:00:27,254 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/75a5a39ee7bf008864556f27870615b9/.tmp/A/6b95983c3e394d52bd279882723afe46, store: [table=TestAcidGuarantees family=A region=75a5a39ee7bf008864556f27870615b9] 2024-11-16T10:00:27,255 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/75a5a39ee7bf008864556f27870615b9/.tmp/A/6b95983c3e394d52bd279882723afe46 is 175, key is test_row_0/A:col10/1731751227211/Put/seqid=0 2024-11-16T10:00:27,257 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=75a5a39ee7bf008864556f27870615b9, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T10:00:27,257 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34428 deadline: 1731751287247, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=75a5a39ee7bf008864556f27870615b9, server=037c58a19ddb,37267,1731751077844 2024-11-16T10:00:27,257 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=75a5a39ee7bf008864556f27870615b9, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T10:00:27,257 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34454 deadline: 1731751287248, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=75a5a39ee7bf008864556f27870615b9, server=037c58a19ddb,37267,1731751077844 2024-11-16T10:00:27,258 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=75a5a39ee7bf008864556f27870615b9, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T10:00:27,258 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34412 deadline: 1731751287254, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=75a5a39ee7bf008864556f27870615b9, server=037c58a19ddb,37267,1731751077844 2024-11-16T10:00:27,258 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731751217321.75a5a39ee7bf008864556f27870615b9., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 , the old value is region=TestAcidGuarantees,,1731751217321.75a5a39ee7bf008864556f27870615b9., hostname=037c58a19ddb,37267,1731751077844, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=75a5a39ee7bf008864556f27870615b9, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:00:27,258 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731751217321.75a5a39ee7bf008864556f27870615b9., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=75a5a39ee7bf008864556f27870615b9, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:00:27,258 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731751217321.75a5a39ee7bf008864556f27870615b9., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 because the exception is null or not the one we care about 2024-11-16T10:00:27,258 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731751217321.75a5a39ee7bf008864556f27870615b9., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 , the old value is region=TestAcidGuarantees,,1731751217321.75a5a39ee7bf008864556f27870615b9., hostname=037c58a19ddb,37267,1731751077844, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=75a5a39ee7bf008864556f27870615b9, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:00:27,258 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731751217321.75a5a39ee7bf008864556f27870615b9., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=75a5a39ee7bf008864556f27870615b9, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:00:27,258 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731751217321.75a5a39ee7bf008864556f27870615b9., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 because the exception is null or not the one we care about 2024-11-16T10:00:27,259 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731751217321.75a5a39ee7bf008864556f27870615b9., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 , the old value is region=TestAcidGuarantees,,1731751217321.75a5a39ee7bf008864556f27870615b9., hostname=037c58a19ddb,37267,1731751077844, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=75a5a39ee7bf008864556f27870615b9, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:00:27,259 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731751217321.75a5a39ee7bf008864556f27870615b9., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=75a5a39ee7bf008864556f27870615b9, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:00:27,259 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731751217321.75a5a39ee7bf008864556f27870615b9., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 because the exception is null or not the one we care about 2024-11-16T10:00:27,262 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=75a5a39ee7bf008864556f27870615b9, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T10:00:27,262 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34392 deadline: 1731751287256, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=75a5a39ee7bf008864556f27870615b9, server=037c58a19ddb,37267,1731751077844 2024-11-16T10:00:27,262 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=75a5a39ee7bf008864556f27870615b9, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T10:00:27,262 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34430 deadline: 1731751287257, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=75a5a39ee7bf008864556f27870615b9, server=037c58a19ddb,37267,1731751077844 2024-11-16T10:00:27,262 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731751217321.75a5a39ee7bf008864556f27870615b9., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 , the old value is region=TestAcidGuarantees,,1731751217321.75a5a39ee7bf008864556f27870615b9., hostname=037c58a19ddb,37267,1731751077844, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=75a5a39ee7bf008864556f27870615b9, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:00:27,262 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731751217321.75a5a39ee7bf008864556f27870615b9., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=75a5a39ee7bf008864556f27870615b9, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:00:27,262 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731751217321.75a5a39ee7bf008864556f27870615b9., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 because the exception is null or not the one we care about 2024-11-16T10:00:27,263 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731751217321.75a5a39ee7bf008864556f27870615b9., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 , the old value is region=TestAcidGuarantees,,1731751217321.75a5a39ee7bf008864556f27870615b9., hostname=037c58a19ddb,37267,1731751077844, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=75a5a39ee7bf008864556f27870615b9, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:00:27,263 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731751217321.75a5a39ee7bf008864556f27870615b9., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=75a5a39ee7bf008864556f27870615b9, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:00:27,263 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731751217321.75a5a39ee7bf008864556f27870615b9., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 because the exception is null or not the one we care about 2024-11-16T10:00:27,270 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37591 is added to blk_1073742400_1576 (size=48289) 2024-11-16T10:00:27,271 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=131, memsize=22.4 K, hasBloomFilter=true, into tmp file hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/75a5a39ee7bf008864556f27870615b9/.tmp/A/6b95983c3e394d52bd279882723afe46 2024-11-16T10:00:27,281 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/75a5a39ee7bf008864556f27870615b9/.tmp/B/a1291712189d4c8882e6915360928d4f is 50, key is test_row_0/B:col10/1731751227211/Put/seqid=0 2024-11-16T10:00:27,298 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37591 is added to blk_1073742401_1577 (size=12101) 2024-11-16T10:00:27,299 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=131 (bloomFilter=true), to=hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/75a5a39ee7bf008864556f27870615b9/.tmp/B/a1291712189d4c8882e6915360928d4f 2024-11-16T10:00:27,309 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/75a5a39ee7bf008864556f27870615b9/.tmp/C/659904f59b7c4a5aa6ff2a8d9f0a61c3 is 50, key is test_row_0/C:col10/1731751227211/Put/seqid=0 2024-11-16T10:00:27,325 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37591 is added to blk_1073742402_1578 (size=12101) 2024-11-16T10:00:27,327 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=131 (bloomFilter=true), to=hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/75a5a39ee7bf008864556f27870615b9/.tmp/C/659904f59b7c4a5aa6ff2a8d9f0a61c3 2024-11-16T10:00:27,333 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/75a5a39ee7bf008864556f27870615b9/.tmp/A/6b95983c3e394d52bd279882723afe46 as hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/75a5a39ee7bf008864556f27870615b9/A/6b95983c3e394d52bd279882723afe46 2024-11-16T10:00:27,338 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/75a5a39ee7bf008864556f27870615b9/A/6b95983c3e394d52bd279882723afe46, entries=250, sequenceid=131, filesize=47.2 K 2024-11-16T10:00:27,338 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/75a5a39ee7bf008864556f27870615b9/.tmp/B/a1291712189d4c8882e6915360928d4f as hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/75a5a39ee7bf008864556f27870615b9/B/a1291712189d4c8882e6915360928d4f 2024-11-16T10:00:27,352 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/75a5a39ee7bf008864556f27870615b9/B/a1291712189d4c8882e6915360928d4f, entries=150, sequenceid=131, filesize=11.8 K 2024-11-16T10:00:27,353 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/75a5a39ee7bf008864556f27870615b9/.tmp/C/659904f59b7c4a5aa6ff2a8d9f0a61c3 as hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/75a5a39ee7bf008864556f27870615b9/C/659904f59b7c4a5aa6ff2a8d9f0a61c3 2024-11-16T10:00:27,358 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/75a5a39ee7bf008864556f27870615b9/C/659904f59b7c4a5aa6ff2a8d9f0a61c3, entries=150, sequenceid=131, filesize=11.8 K 2024-11-16T10:00:27,359 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~67.09 KB/68700, heapSize ~176.48 KB/180720, currentSize=134.18 KB/137400 for 75a5a39ee7bf008864556f27870615b9 in 142ms, sequenceid=131, compaction requested=true 2024-11-16T10:00:27,359 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 75a5a39ee7bf008864556f27870615b9: 2024-11-16T10:00:27,359 DEBUG [RS:0;037c58a19ddb:37267-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-16T10:00:27,359 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 75a5a39ee7bf008864556f27870615b9:A, priority=-2147483648, current under compaction store size is 1 2024-11-16T10:00:27,359 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-16T10:00:27,359 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 75a5a39ee7bf008864556f27870615b9:B, priority=-2147483648, current under compaction store size is 2 2024-11-16T10:00:27,359 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-16T10:00:27,359 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 75a5a39ee7bf008864556f27870615b9:C, priority=-2147483648, current under compaction store size is 3 2024-11-16T10:00:27,359 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=2), splitQueue=0 2024-11-16T10:00:27,360 DEBUG [RS:0;037c58a19ddb:37267-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 110405 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-16T10:00:27,360 DEBUG [RS:0;037c58a19ddb:37267-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-16T10:00:27,360 DEBUG [RS:0;037c58a19ddb:37267-shortCompactions-0 {}] regionserver.HStore(1541): 75a5a39ee7bf008864556f27870615b9/A is initiating minor compaction (all files) 2024-11-16T10:00:27,360 INFO [RS:0;037c58a19ddb:37267-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 75a5a39ee7bf008864556f27870615b9/A in TestAcidGuarantees,,1731751217321.75a5a39ee7bf008864556f27870615b9. 2024-11-16T10:00:27,360 INFO [RS:0;037c58a19ddb:37267-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/75a5a39ee7bf008864556f27870615b9/A/c45414a6fed343d78f84a9867a4a9bfc, hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/75a5a39ee7bf008864556f27870615b9/A/a229e16c29e64ca982c262db7c64505c, hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/75a5a39ee7bf008864556f27870615b9/A/6b95983c3e394d52bd279882723afe46] into tmpdir=hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/75a5a39ee7bf008864556f27870615b9/.tmp, totalSize=107.8 K 2024-11-16T10:00:27,360 INFO [RS:0;037c58a19ddb:37267-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731751217321.75a5a39ee7bf008864556f27870615b9. 2024-11-16T10:00:27,360 DEBUG [RS:0;037c58a19ddb:37267-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731751217321.75a5a39ee7bf008864556f27870615b9. files: [hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/75a5a39ee7bf008864556f27870615b9/A/c45414a6fed343d78f84a9867a4a9bfc, hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/75a5a39ee7bf008864556f27870615b9/A/a229e16c29e64ca982c262db7c64505c, hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/75a5a39ee7bf008864556f27870615b9/A/6b95983c3e394d52bd279882723afe46] 2024-11-16T10:00:27,362 DEBUG [RS:0;037c58a19ddb:37267-shortCompactions-0 {}] compactions.Compactor(225): Compacting c45414a6fed343d78f84a9867a4a9bfc, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=95, earliestPutTs=1731751223859 2024-11-16T10:00:27,362 DEBUG [RS:0;037c58a19ddb:37267-shortCompactions-0 {}] compactions.Compactor(225): Compacting a229e16c29e64ca982c262db7c64505c, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=118, earliestPutTs=1731751225020 2024-11-16T10:00:27,362 DEBUG [RS:0;037c58a19ddb:37267-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36309 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-16T10:00:27,363 DEBUG [RS:0;037c58a19ddb:37267-longCompactions-0 {}] regionserver.HStore(1541): 75a5a39ee7bf008864556f27870615b9/B is initiating minor compaction (all files) 2024-11-16T10:00:27,363 DEBUG [RS:0;037c58a19ddb:37267-shortCompactions-0 {}] compactions.Compactor(225): Compacting 6b95983c3e394d52bd279882723afe46, keycount=250, bloomtype=ROW, size=47.2 K, encoding=NONE, compression=NONE, seqNum=131, earliestPutTs=1731751227211 2024-11-16T10:00:27,363 INFO [RS:0;037c58a19ddb:37267-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 75a5a39ee7bf008864556f27870615b9/B in TestAcidGuarantees,,1731751217321.75a5a39ee7bf008864556f27870615b9. 2024-11-16T10:00:27,363 INFO [RS:0;037c58a19ddb:37267-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/75a5a39ee7bf008864556f27870615b9/B/371f5fd3a9984c5a9022b16ad2d69107, hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/75a5a39ee7bf008864556f27870615b9/B/8c92dd9bd0d34f40b1dab0876669e75e, hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/75a5a39ee7bf008864556f27870615b9/B/a1291712189d4c8882e6915360928d4f] into tmpdir=hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/75a5a39ee7bf008864556f27870615b9/.tmp, totalSize=35.5 K 2024-11-16T10:00:27,365 DEBUG [RS:0;037c58a19ddb:37267-longCompactions-0 {}] compactions.Compactor(225): Compacting 371f5fd3a9984c5a9022b16ad2d69107, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=95, earliestPutTs=1731751223859 2024-11-16T10:00:27,366 DEBUG [RS:0;037c58a19ddb:37267-longCompactions-0 {}] compactions.Compactor(225): Compacting 8c92dd9bd0d34f40b1dab0876669e75e, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=118, earliestPutTs=1731751225020 2024-11-16T10:00:27,366 DEBUG [RS:0;037c58a19ddb:37267-longCompactions-0 {}] compactions.Compactor(225): Compacting a1291712189d4c8882e6915360928d4f, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=131, earliestPutTs=1731751227211 2024-11-16T10:00:27,367 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-16T10:00:27,367 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 75a5a39ee7bf008864556f27870615b9 3/3 column families, dataSize=140.89 KB heapSize=369.89 KB 2024-11-16T10:00:27,367 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] regionserver.HRegion(8855): Flush requested on 75a5a39ee7bf008864556f27870615b9 2024-11-16T10:00:27,372 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=37267 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=157 2024-11-16T10:00:27,373 DEBUG [RS_FLUSH_OPERATIONS-regionserver/037c58a19ddb:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731751217321.75a5a39ee7bf008864556f27870615b9. 2024-11-16T10:00:27,373 DEBUG [RS_FLUSH_OPERATIONS-regionserver/037c58a19ddb:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731751217321.75a5a39ee7bf008864556f27870615b9. as already flushing 2024-11-16T10:00:27,373 DEBUG [RS_FLUSH_OPERATIONS-regionserver/037c58a19ddb:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731751217321.75a5a39ee7bf008864556f27870615b9. 2024-11-16T10:00:27,373 ERROR [RS_FLUSH_OPERATIONS-regionserver/037c58a19ddb:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] handler.RSProcedureHandler(58): pid=157 java.io.IOException: Unable to complete flush {ENCODED => 75a5a39ee7bf008864556f27870615b9, NAME => 'TestAcidGuarantees,,1731751217321.75a5a39ee7bf008864556f27870615b9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T10:00:27,373 DEBUG [RS_FLUSH_OPERATIONS-regionserver/037c58a19ddb:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=157 java.io.IOException: Unable to complete flush {ENCODED => 75a5a39ee7bf008864556f27870615b9, NAME => 'TestAcidGuarantees,,1731751217321.75a5a39ee7bf008864556f27870615b9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T10:00:27,374 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37877 {}] master.HMaster(4177): Remote procedure failed, pid=157 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 75a5a39ee7bf008864556f27870615b9, NAME => 'TestAcidGuarantees,,1731751217321.75a5a39ee7bf008864556f27870615b9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 75a5a39ee7bf008864556f27870615b9, NAME => 'TestAcidGuarantees,,1731751217321.75a5a39ee7bf008864556f27870615b9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T10:00:27,381 INFO [RS:0;037c58a19ddb:37267-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=75a5a39ee7bf008864556f27870615b9] 2024-11-16T10:00:27,384 INFO [RS:0;037c58a19ddb:37267-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 75a5a39ee7bf008864556f27870615b9#B#compaction#494 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-16T10:00:27,385 DEBUG [RS:0;037c58a19ddb:37267-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/75a5a39ee7bf008864556f27870615b9/.tmp/B/53b261f643444e2f9eeda0200e77f83c is 50, key is test_row_0/B:col10/1731751227211/Put/seqid=0 2024-11-16T10:00:27,385 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37877 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=156 2024-11-16T10:00:27,387 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=75a5a39ee7bf008864556f27870615b9, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T10:00:27,387 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34454 deadline: 1731751287379, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=75a5a39ee7bf008864556f27870615b9, server=037c58a19ddb,37267,1731751077844 2024-11-16T10:00:27,388 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=75a5a39ee7bf008864556f27870615b9, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T10:00:27,388 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731751217321.75a5a39ee7bf008864556f27870615b9., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 , the old value is region=TestAcidGuarantees,,1731751217321.75a5a39ee7bf008864556f27870615b9., hostname=037c58a19ddb,37267,1731751077844, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=75a5a39ee7bf008864556f27870615b9, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:00:27,388 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34392 deadline: 1731751287381, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=75a5a39ee7bf008864556f27870615b9, server=037c58a19ddb,37267,1731751077844 2024-11-16T10:00:27,388 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731751217321.75a5a39ee7bf008864556f27870615b9., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=75a5a39ee7bf008864556f27870615b9, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:00:27,388 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731751217321.75a5a39ee7bf008864556f27870615b9., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 because the exception is null or not the one we care about 2024-11-16T10:00:27,388 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731751217321.75a5a39ee7bf008864556f27870615b9., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 , the old value is region=TestAcidGuarantees,,1731751217321.75a5a39ee7bf008864556f27870615b9., hostname=037c58a19ddb,37267,1731751077844, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=75a5a39ee7bf008864556f27870615b9, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:00:27,388 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731751217321.75a5a39ee7bf008864556f27870615b9., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=75a5a39ee7bf008864556f27870615b9, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:00:27,388 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731751217321.75a5a39ee7bf008864556f27870615b9., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 because the exception is null or not the one we care about 2024-11-16T10:00:27,390 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=75a5a39ee7bf008864556f27870615b9, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T10:00:27,390 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34430 deadline: 1731751287383, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=75a5a39ee7bf008864556f27870615b9, server=037c58a19ddb,37267,1731751077844 2024-11-16T10:00:27,390 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=75a5a39ee7bf008864556f27870615b9, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T10:00:27,390 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34428 deadline: 1731751287384, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=75a5a39ee7bf008864556f27870615b9, server=037c58a19ddb,37267,1731751077844 2024-11-16T10:00:27,390 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=75a5a39ee7bf008864556f27870615b9, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T10:00:27,390 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731751217321.75a5a39ee7bf008864556f27870615b9., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 , the old value is region=TestAcidGuarantees,,1731751217321.75a5a39ee7bf008864556f27870615b9., hostname=037c58a19ddb,37267,1731751077844, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=75a5a39ee7bf008864556f27870615b9, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:00:27,391 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731751217321.75a5a39ee7bf008864556f27870615b9., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=75a5a39ee7bf008864556f27870615b9, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:00:27,391 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731751217321.75a5a39ee7bf008864556f27870615b9., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 , the old value is region=TestAcidGuarantees,,1731751217321.75a5a39ee7bf008864556f27870615b9., hostname=037c58a19ddb,37267,1731751077844, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=75a5a39ee7bf008864556f27870615b9, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:00:27,391 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34412 deadline: 1731751287386, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=75a5a39ee7bf008864556f27870615b9, server=037c58a19ddb,37267,1731751077844 2024-11-16T10:00:27,391 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731751217321.75a5a39ee7bf008864556f27870615b9., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 because the exception is null or not the one we care about 2024-11-16T10:00:27,391 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731751217321.75a5a39ee7bf008864556f27870615b9., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=75a5a39ee7bf008864556f27870615b9, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:00:27,391 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731751217321.75a5a39ee7bf008864556f27870615b9., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 because the exception is null or not the one we care about 2024-11-16T10:00:27,391 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731751217321.75a5a39ee7bf008864556f27870615b9., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 , the old value is region=TestAcidGuarantees,,1731751217321.75a5a39ee7bf008864556f27870615b9., hostname=037c58a19ddb,37267,1731751077844, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=75a5a39ee7bf008864556f27870615b9, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:00:27,391 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731751217321.75a5a39ee7bf008864556f27870615b9., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=75a5a39ee7bf008864556f27870615b9, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:00:27,391 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731751217321.75a5a39ee7bf008864556f27870615b9., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 because the exception is null or not the one we care about 2024-11-16T10:00:27,392 DEBUG [RS:0;037c58a19ddb:37267-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241116aec62e8039e34fb1ad05c973a21a9eac_75a5a39ee7bf008864556f27870615b9 store=[table=TestAcidGuarantees family=A region=75a5a39ee7bf008864556f27870615b9] 2024-11-16T10:00:27,395 DEBUG [RS:0;037c58a19ddb:37267-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241116aec62e8039e34fb1ad05c973a21a9eac_75a5a39ee7bf008864556f27870615b9, store=[table=TestAcidGuarantees family=A region=75a5a39ee7bf008864556f27870615b9] 2024-11-16T10:00:27,395 DEBUG [RS:0;037c58a19ddb:37267-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241116aec62e8039e34fb1ad05c973a21a9eac_75a5a39ee7bf008864556f27870615b9 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=75a5a39ee7bf008864556f27870615b9] 2024-11-16T10:00:27,407 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241116f7c8313634574e3688e4c0f01e940f78_75a5a39ee7bf008864556f27870615b9 is 50, key is test_row_0/A:col10/1731751227246/Put/seqid=0 2024-11-16T10:00:27,434 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37591 is added to blk_1073742403_1579 (size=12409) 2024-11-16T10:00:27,441 DEBUG [RS:0;037c58a19ddb:37267-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/75a5a39ee7bf008864556f27870615b9/.tmp/B/53b261f643444e2f9eeda0200e77f83c as hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/75a5a39ee7bf008864556f27870615b9/B/53b261f643444e2f9eeda0200e77f83c 2024-11-16T10:00:27,448 INFO [RS:0;037c58a19ddb:37267-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 75a5a39ee7bf008864556f27870615b9/B of 75a5a39ee7bf008864556f27870615b9 into 53b261f643444e2f9eeda0200e77f83c(size=12.1 K), total size for store is 12.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-16T10:00:27,448 DEBUG [RS:0;037c58a19ddb:37267-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 75a5a39ee7bf008864556f27870615b9: 2024-11-16T10:00:27,448 INFO [RS:0;037c58a19ddb:37267-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731751217321.75a5a39ee7bf008864556f27870615b9., storeName=75a5a39ee7bf008864556f27870615b9/B, priority=13, startTime=1731751227359; duration=0sec 2024-11-16T10:00:27,448 DEBUG [RS:0;037c58a19ddb:37267-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-16T10:00:27,448 DEBUG [RS:0;037c58a19ddb:37267-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 75a5a39ee7bf008864556f27870615b9:B 2024-11-16T10:00:27,448 DEBUG [RS:0;037c58a19ddb:37267-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-16T10:00:27,449 DEBUG [RS:0;037c58a19ddb:37267-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36309 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-16T10:00:27,449 DEBUG [RS:0;037c58a19ddb:37267-longCompactions-0 {}] regionserver.HStore(1541): 75a5a39ee7bf008864556f27870615b9/C is initiating minor compaction (all files) 2024-11-16T10:00:27,449 INFO [RS:0;037c58a19ddb:37267-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 75a5a39ee7bf008864556f27870615b9/C in TestAcidGuarantees,,1731751217321.75a5a39ee7bf008864556f27870615b9. 2024-11-16T10:00:27,449 INFO [RS:0;037c58a19ddb:37267-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/75a5a39ee7bf008864556f27870615b9/C/c36f6d576030452db8db9b4cf83d536e, hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/75a5a39ee7bf008864556f27870615b9/C/6b0168243cd5427aa60eaa61687c5d65, hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/75a5a39ee7bf008864556f27870615b9/C/659904f59b7c4a5aa6ff2a8d9f0a61c3] into tmpdir=hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/75a5a39ee7bf008864556f27870615b9/.tmp, totalSize=35.5 K 2024-11-16T10:00:27,450 DEBUG [RS:0;037c58a19ddb:37267-longCompactions-0 {}] compactions.Compactor(225): Compacting c36f6d576030452db8db9b4cf83d536e, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=95, earliestPutTs=1731751223859 2024-11-16T10:00:27,450 DEBUG [RS:0;037c58a19ddb:37267-longCompactions-0 {}] compactions.Compactor(225): Compacting 6b0168243cd5427aa60eaa61687c5d65, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=118, earliestPutTs=1731751225020 2024-11-16T10:00:27,451 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37591 is added to blk_1073742404_1580 (size=4469) 2024-11-16T10:00:27,451 DEBUG [RS:0;037c58a19ddb:37267-longCompactions-0 {}] compactions.Compactor(225): Compacting 659904f59b7c4a5aa6ff2a8d9f0a61c3, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=131, earliestPutTs=1731751227211 2024-11-16T10:00:27,454 INFO [RS:0;037c58a19ddb:37267-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 75a5a39ee7bf008864556f27870615b9#A#compaction#493 average throughput is 0.33 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-16T10:00:27,454 DEBUG [RS:0;037c58a19ddb:37267-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/75a5a39ee7bf008864556f27870615b9/.tmp/A/2567f08bfa9a407898aa25582d9a0dff is 175, key is test_row_0/A:col10/1731751227211/Put/seqid=0 2024-11-16T10:00:27,459 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37591 is added to blk_1073742405_1581 (size=14794) 2024-11-16T10:00:27,460 INFO [RS:0;037c58a19ddb:37267-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 75a5a39ee7bf008864556f27870615b9#C#compaction#496 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-16T10:00:27,461 DEBUG [RS:0;037c58a19ddb:37267-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/75a5a39ee7bf008864556f27870615b9/.tmp/C/22cf205d63134391b67b77b703ef814d is 50, key is test_row_0/C:col10/1731751227211/Put/seqid=0 2024-11-16T10:00:27,477 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37591 is added to blk_1073742406_1582 (size=31363) 2024-11-16T10:00:27,482 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37591 is added to blk_1073742407_1583 (size=12409) 2024-11-16T10:00:27,487 DEBUG [RS:0;037c58a19ddb:37267-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/75a5a39ee7bf008864556f27870615b9/.tmp/C/22cf205d63134391b67b77b703ef814d as hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/75a5a39ee7bf008864556f27870615b9/C/22cf205d63134391b67b77b703ef814d 2024-11-16T10:00:27,492 INFO [RS:0;037c58a19ddb:37267-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 75a5a39ee7bf008864556f27870615b9/C of 75a5a39ee7bf008864556f27870615b9 into 22cf205d63134391b67b77b703ef814d(size=12.1 K), total size for store is 12.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-16T10:00:27,492 DEBUG [RS:0;037c58a19ddb:37267-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 75a5a39ee7bf008864556f27870615b9: 2024-11-16T10:00:27,492 INFO [RS:0;037c58a19ddb:37267-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731751217321.75a5a39ee7bf008864556f27870615b9., storeName=75a5a39ee7bf008864556f27870615b9/C, priority=13, startTime=1731751227359; duration=0sec 2024-11-16T10:00:27,492 DEBUG [RS:0;037c58a19ddb:37267-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-16T10:00:27,492 DEBUG [RS:0;037c58a19ddb:37267-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 75a5a39ee7bf008864556f27870615b9:C 2024-11-16T10:00:27,493 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=75a5a39ee7bf008864556f27870615b9, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T10:00:27,494 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34454 deadline: 1731751287493, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=75a5a39ee7bf008864556f27870615b9, server=037c58a19ddb,37267,1731751077844 2024-11-16T10:00:27,494 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=75a5a39ee7bf008864556f27870615b9, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T10:00:27,494 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34392 deadline: 1731751287493, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=75a5a39ee7bf008864556f27870615b9, server=037c58a19ddb,37267,1731751077844 2024-11-16T10:00:27,494 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731751217321.75a5a39ee7bf008864556f27870615b9., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 , the old value is region=TestAcidGuarantees,,1731751217321.75a5a39ee7bf008864556f27870615b9., hostname=037c58a19ddb,37267,1731751077844, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=75a5a39ee7bf008864556f27870615b9, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:00:27,494 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731751217321.75a5a39ee7bf008864556f27870615b9., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=75a5a39ee7bf008864556f27870615b9, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:00:27,494 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731751217321.75a5a39ee7bf008864556f27870615b9., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 , the old value is region=TestAcidGuarantees,,1731751217321.75a5a39ee7bf008864556f27870615b9., hostname=037c58a19ddb,37267,1731751077844, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=75a5a39ee7bf008864556f27870615b9, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:00:27,494 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731751217321.75a5a39ee7bf008864556f27870615b9., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 because the exception is null or not the one we care about 2024-11-16T10:00:27,494 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=75a5a39ee7bf008864556f27870615b9, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T10:00:27,494 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731751217321.75a5a39ee7bf008864556f27870615b9., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=75a5a39ee7bf008864556f27870615b9, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:00:27,494 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731751217321.75a5a39ee7bf008864556f27870615b9., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 because the exception is null or not the one we care about 2024-11-16T10:00:27,494 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34412 deadline: 1731751287493, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=75a5a39ee7bf008864556f27870615b9, server=037c58a19ddb,37267,1731751077844 2024-11-16T10:00:27,495 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731751217321.75a5a39ee7bf008864556f27870615b9., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 , the old value is region=TestAcidGuarantees,,1731751217321.75a5a39ee7bf008864556f27870615b9., hostname=037c58a19ddb,37267,1731751077844, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=75a5a39ee7bf008864556f27870615b9, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:00:27,495 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731751217321.75a5a39ee7bf008864556f27870615b9., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=75a5a39ee7bf008864556f27870615b9, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:00:27,495 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731751217321.75a5a39ee7bf008864556f27870615b9., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 because the exception is null or not the one we care about 2024-11-16T10:00:27,498 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=75a5a39ee7bf008864556f27870615b9, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T10:00:27,498 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34430 deadline: 1731751287494, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=75a5a39ee7bf008864556f27870615b9, server=037c58a19ddb,37267,1731751077844 2024-11-16T10:00:27,498 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=75a5a39ee7bf008864556f27870615b9, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T10:00:27,498 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34428 deadline: 1731751287494, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=75a5a39ee7bf008864556f27870615b9, server=037c58a19ddb,37267,1731751077844 2024-11-16T10:00:27,498 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731751217321.75a5a39ee7bf008864556f27870615b9., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 , the old value is region=TestAcidGuarantees,,1731751217321.75a5a39ee7bf008864556f27870615b9., hostname=037c58a19ddb,37267,1731751077844, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=75a5a39ee7bf008864556f27870615b9, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:00:27,498 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731751217321.75a5a39ee7bf008864556f27870615b9., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=75a5a39ee7bf008864556f27870615b9, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:00:27,498 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731751217321.75a5a39ee7bf008864556f27870615b9., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 because the exception is null or not the one we care about 2024-11-16T10:00:27,498 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731751217321.75a5a39ee7bf008864556f27870615b9., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 , the old value is region=TestAcidGuarantees,,1731751217321.75a5a39ee7bf008864556f27870615b9., hostname=037c58a19ddb,37267,1731751077844, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=75a5a39ee7bf008864556f27870615b9, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:00:27,498 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731751217321.75a5a39ee7bf008864556f27870615b9., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=75a5a39ee7bf008864556f27870615b9, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:00:27,498 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731751217321.75a5a39ee7bf008864556f27870615b9., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 because the exception is null or not the one we care about 2024-11-16T10:00:27,527 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=37267 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=157 2024-11-16T10:00:27,527 DEBUG [RS_FLUSH_OPERATIONS-regionserver/037c58a19ddb:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731751217321.75a5a39ee7bf008864556f27870615b9. 2024-11-16T10:00:27,527 DEBUG [RS_FLUSH_OPERATIONS-regionserver/037c58a19ddb:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731751217321.75a5a39ee7bf008864556f27870615b9. as already flushing 2024-11-16T10:00:27,527 DEBUG [RS_FLUSH_OPERATIONS-regionserver/037c58a19ddb:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731751217321.75a5a39ee7bf008864556f27870615b9. 2024-11-16T10:00:27,527 ERROR [RS_FLUSH_OPERATIONS-regionserver/037c58a19ddb:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] handler.RSProcedureHandler(58): pid=157 java.io.IOException: Unable to complete flush {ENCODED => 75a5a39ee7bf008864556f27870615b9, NAME => 'TestAcidGuarantees,,1731751217321.75a5a39ee7bf008864556f27870615b9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T10:00:27,527 DEBUG [RS_FLUSH_OPERATIONS-regionserver/037c58a19ddb:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=157 java.io.IOException: Unable to complete flush {ENCODED => 75a5a39ee7bf008864556f27870615b9, NAME => 'TestAcidGuarantees,,1731751217321.75a5a39ee7bf008864556f27870615b9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T10:00:27,528 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37877 {}] master.HMaster(4177): Remote procedure failed, pid=157 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 75a5a39ee7bf008864556f27870615b9, NAME => 'TestAcidGuarantees,,1731751217321.75a5a39ee7bf008864556f27870615b9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 75a5a39ee7bf008864556f27870615b9, NAME => 'TestAcidGuarantees,,1731751217321.75a5a39ee7bf008864556f27870615b9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T10:00:27,539 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_TestAcidGuarantees 2024-11-16T10:00:27,539 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_default_table_TestAcidGuarantees Metrics about Tables on a single HBase RegionServer 2024-11-16T10:00:27,680 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=37267 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=157 2024-11-16T10:00:27,680 DEBUG [RS_FLUSH_OPERATIONS-regionserver/037c58a19ddb:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731751217321.75a5a39ee7bf008864556f27870615b9. 2024-11-16T10:00:27,680 DEBUG [RS_FLUSH_OPERATIONS-regionserver/037c58a19ddb:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731751217321.75a5a39ee7bf008864556f27870615b9. as already flushing 2024-11-16T10:00:27,680 DEBUG [RS_FLUSH_OPERATIONS-regionserver/037c58a19ddb:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731751217321.75a5a39ee7bf008864556f27870615b9. 2024-11-16T10:00:27,680 ERROR [RS_FLUSH_OPERATIONS-regionserver/037c58a19ddb:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] handler.RSProcedureHandler(58): pid=157 java.io.IOException: Unable to complete flush {ENCODED => 75a5a39ee7bf008864556f27870615b9, NAME => 'TestAcidGuarantees,,1731751217321.75a5a39ee7bf008864556f27870615b9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T10:00:27,680 DEBUG [RS_FLUSH_OPERATIONS-regionserver/037c58a19ddb:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=157 java.io.IOException: Unable to complete flush {ENCODED => 75a5a39ee7bf008864556f27870615b9, NAME => 'TestAcidGuarantees,,1731751217321.75a5a39ee7bf008864556f27870615b9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T10:00:27,681 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37877 {}] master.HMaster(4177): Remote procedure failed, pid=157 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 75a5a39ee7bf008864556f27870615b9, NAME => 'TestAcidGuarantees,,1731751217321.75a5a39ee7bf008864556f27870615b9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 75a5a39ee7bf008864556f27870615b9, NAME => 'TestAcidGuarantees,,1731751217321.75a5a39ee7bf008864556f27870615b9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T10:00:27,693 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37877 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=156 2024-11-16T10:00:27,705 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=75a5a39ee7bf008864556f27870615b9, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T10:00:27,705 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34454 deadline: 1731751287703, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=75a5a39ee7bf008864556f27870615b9, server=037c58a19ddb,37267,1731751077844 2024-11-16T10:00:27,705 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=75a5a39ee7bf008864556f27870615b9, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T10:00:27,705 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34392 deadline: 1731751287703, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=75a5a39ee7bf008864556f27870615b9, server=037c58a19ddb,37267,1731751077844 2024-11-16T10:00:27,705 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731751217321.75a5a39ee7bf008864556f27870615b9., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 , the old value is region=TestAcidGuarantees,,1731751217321.75a5a39ee7bf008864556f27870615b9., hostname=037c58a19ddb,37267,1731751077844, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=75a5a39ee7bf008864556f27870615b9, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:00:27,705 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731751217321.75a5a39ee7bf008864556f27870615b9., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=75a5a39ee7bf008864556f27870615b9, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:00:27,705 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731751217321.75a5a39ee7bf008864556f27870615b9., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 because the exception is null or not the one we care about 2024-11-16T10:00:27,706 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=75a5a39ee7bf008864556f27870615b9, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T10:00:27,706 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34412 deadline: 1731751287703, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=75a5a39ee7bf008864556f27870615b9, server=037c58a19ddb,37267,1731751077844 2024-11-16T10:00:27,706 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731751217321.75a5a39ee7bf008864556f27870615b9., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 , the old value is region=TestAcidGuarantees,,1731751217321.75a5a39ee7bf008864556f27870615b9., hostname=037c58a19ddb,37267,1731751077844, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=75a5a39ee7bf008864556f27870615b9, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:00:27,706 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731751217321.75a5a39ee7bf008864556f27870615b9., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=75a5a39ee7bf008864556f27870615b9, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:00:27,706 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731751217321.75a5a39ee7bf008864556f27870615b9., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 because the exception is null or not the one we care about 2024-11-16T10:00:27,706 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=75a5a39ee7bf008864556f27870615b9, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T10:00:27,706 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731751217321.75a5a39ee7bf008864556f27870615b9., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 , the old value is region=TestAcidGuarantees,,1731751217321.75a5a39ee7bf008864556f27870615b9., hostname=037c58a19ddb,37267,1731751077844, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=75a5a39ee7bf008864556f27870615b9, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:00:27,706 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731751217321.75a5a39ee7bf008864556f27870615b9., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=75a5a39ee7bf008864556f27870615b9, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:00:27,706 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34430 deadline: 1731751287703, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=75a5a39ee7bf008864556f27870615b9, server=037c58a19ddb,37267,1731751077844 2024-11-16T10:00:27,706 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731751217321.75a5a39ee7bf008864556f27870615b9., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 because the exception is null or not the one we care about 2024-11-16T10:00:27,706 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=75a5a39ee7bf008864556f27870615b9, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T10:00:27,706 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731751217321.75a5a39ee7bf008864556f27870615b9., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 , the old value is region=TestAcidGuarantees,,1731751217321.75a5a39ee7bf008864556f27870615b9., hostname=037c58a19ddb,37267,1731751077844, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=75a5a39ee7bf008864556f27870615b9, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:00:27,706 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34428 deadline: 1731751287703, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=75a5a39ee7bf008864556f27870615b9, server=037c58a19ddb,37267,1731751077844 2024-11-16T10:00:27,706 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731751217321.75a5a39ee7bf008864556f27870615b9., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=75a5a39ee7bf008864556f27870615b9, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:00:27,706 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731751217321.75a5a39ee7bf008864556f27870615b9., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 because the exception is null or not the one we care about 2024-11-16T10:00:27,706 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731751217321.75a5a39ee7bf008864556f27870615b9., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 , the old value is region=TestAcidGuarantees,,1731751217321.75a5a39ee7bf008864556f27870615b9., hostname=037c58a19ddb,37267,1731751077844, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=75a5a39ee7bf008864556f27870615b9, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:00:27,706 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731751217321.75a5a39ee7bf008864556f27870615b9., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=75a5a39ee7bf008864556f27870615b9, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:00:27,706 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731751217321.75a5a39ee7bf008864556f27870615b9., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 because the exception is null or not the one we care about 2024-11-16T10:00:27,833 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=37267 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=157 2024-11-16T10:00:27,833 DEBUG [RS_FLUSH_OPERATIONS-regionserver/037c58a19ddb:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731751217321.75a5a39ee7bf008864556f27870615b9. 2024-11-16T10:00:27,833 DEBUG [RS_FLUSH_OPERATIONS-regionserver/037c58a19ddb:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731751217321.75a5a39ee7bf008864556f27870615b9. as already flushing 2024-11-16T10:00:27,833 DEBUG [RS_FLUSH_OPERATIONS-regionserver/037c58a19ddb:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731751217321.75a5a39ee7bf008864556f27870615b9. 2024-11-16T10:00:27,833 ERROR [RS_FLUSH_OPERATIONS-regionserver/037c58a19ddb:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] handler.RSProcedureHandler(58): pid=157 java.io.IOException: Unable to complete flush {ENCODED => 75a5a39ee7bf008864556f27870615b9, NAME => 'TestAcidGuarantees,,1731751217321.75a5a39ee7bf008864556f27870615b9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T10:00:27,833 DEBUG [RS_FLUSH_OPERATIONS-regionserver/037c58a19ddb:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=157 java.io.IOException: Unable to complete flush {ENCODED => 75a5a39ee7bf008864556f27870615b9, NAME => 'TestAcidGuarantees,,1731751217321.75a5a39ee7bf008864556f27870615b9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T10:00:27,834 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37877 {}] master.HMaster(4177): Remote procedure failed, pid=157 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 75a5a39ee7bf008864556f27870615b9, NAME => 'TestAcidGuarantees,,1731751217321.75a5a39ee7bf008864556f27870615b9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 75a5a39ee7bf008864556f27870615b9, NAME => 'TestAcidGuarantees,,1731751217321.75a5a39ee7bf008864556f27870615b9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T10:00:27,862 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:27,865 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241116f7c8313634574e3688e4c0f01e940f78_75a5a39ee7bf008864556f27870615b9 to hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241116f7c8313634574e3688e4c0f01e940f78_75a5a39ee7bf008864556f27870615b9 2024-11-16T10:00:27,868 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/75a5a39ee7bf008864556f27870615b9/.tmp/A/80e1e8f18c254519b6be286bcb29f3db, store: [table=TestAcidGuarantees family=A region=75a5a39ee7bf008864556f27870615b9] 2024-11-16T10:00:27,869 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/75a5a39ee7bf008864556f27870615b9/.tmp/A/80e1e8f18c254519b6be286bcb29f3db is 175, key is test_row_0/A:col10/1731751227246/Put/seqid=0 2024-11-16T10:00:27,872 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37591 is added to blk_1073742408_1584 (size=39749) 2024-11-16T10:00:27,880 DEBUG [RS:0;037c58a19ddb:37267-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/75a5a39ee7bf008864556f27870615b9/.tmp/A/2567f08bfa9a407898aa25582d9a0dff as hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/75a5a39ee7bf008864556f27870615b9/A/2567f08bfa9a407898aa25582d9a0dff 2024-11-16T10:00:27,884 INFO [RS:0;037c58a19ddb:37267-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 75a5a39ee7bf008864556f27870615b9/A of 75a5a39ee7bf008864556f27870615b9 into 2567f08bfa9a407898aa25582d9a0dff(size=30.6 K), total size for store is 30.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-16T10:00:27,884 DEBUG [RS:0;037c58a19ddb:37267-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 75a5a39ee7bf008864556f27870615b9: 2024-11-16T10:00:27,884 INFO [RS:0;037c58a19ddb:37267-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731751217321.75a5a39ee7bf008864556f27870615b9., storeName=75a5a39ee7bf008864556f27870615b9/A, priority=13, startTime=1731751227359; duration=0sec 2024-11-16T10:00:27,885 DEBUG [RS:0;037c58a19ddb:37267-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-16T10:00:27,885 DEBUG [RS:0;037c58a19ddb:37267-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 75a5a39ee7bf008864556f27870615b9:A 2024-11-16T10:00:27,986 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=37267 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=157 2024-11-16T10:00:27,986 DEBUG [RS_FLUSH_OPERATIONS-regionserver/037c58a19ddb:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731751217321.75a5a39ee7bf008864556f27870615b9. 2024-11-16T10:00:27,986 DEBUG [RS_FLUSH_OPERATIONS-regionserver/037c58a19ddb:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731751217321.75a5a39ee7bf008864556f27870615b9. as already flushing 2024-11-16T10:00:27,986 DEBUG [RS_FLUSH_OPERATIONS-regionserver/037c58a19ddb:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731751217321.75a5a39ee7bf008864556f27870615b9. 2024-11-16T10:00:27,986 ERROR [RS_FLUSH_OPERATIONS-regionserver/037c58a19ddb:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] handler.RSProcedureHandler(58): pid=157 java.io.IOException: Unable to complete flush {ENCODED => 75a5a39ee7bf008864556f27870615b9, NAME => 'TestAcidGuarantees,,1731751217321.75a5a39ee7bf008864556f27870615b9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T10:00:27,986 DEBUG [RS_FLUSH_OPERATIONS-regionserver/037c58a19ddb:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=157 java.io.IOException: Unable to complete flush {ENCODED => 75a5a39ee7bf008864556f27870615b9, NAME => 'TestAcidGuarantees,,1731751217321.75a5a39ee7bf008864556f27870615b9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T10:00:27,987 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37877 {}] master.HMaster(4177): Remote procedure failed, pid=157 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 75a5a39ee7bf008864556f27870615b9, NAME => 'TestAcidGuarantees,,1731751217321.75a5a39ee7bf008864556f27870615b9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 75a5a39ee7bf008864556f27870615b9, NAME => 'TestAcidGuarantees,,1731751217321.75a5a39ee7bf008864556f27870615b9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T10:00:28,015 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=75a5a39ee7bf008864556f27870615b9, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T10:00:28,015 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34454 deadline: 1731751288013, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=75a5a39ee7bf008864556f27870615b9, server=037c58a19ddb,37267,1731751077844 2024-11-16T10:00:28,015 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=75a5a39ee7bf008864556f27870615b9, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T10:00:28,015 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731751217321.75a5a39ee7bf008864556f27870615b9., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 , the old value is region=TestAcidGuarantees,,1731751217321.75a5a39ee7bf008864556f27870615b9., hostname=037c58a19ddb,37267,1731751077844, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=75a5a39ee7bf008864556f27870615b9, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:00:28,015 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731751217321.75a5a39ee7bf008864556f27870615b9., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=75a5a39ee7bf008864556f27870615b9, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:00:28,015 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34392 deadline: 1731751288013, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=75a5a39ee7bf008864556f27870615b9, server=037c58a19ddb,37267,1731751077844 2024-11-16T10:00:28,015 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731751217321.75a5a39ee7bf008864556f27870615b9., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 because the exception is null or not the one we care about 2024-11-16T10:00:28,016 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731751217321.75a5a39ee7bf008864556f27870615b9., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 , the old value is region=TestAcidGuarantees,,1731751217321.75a5a39ee7bf008864556f27870615b9., hostname=037c58a19ddb,37267,1731751077844, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=75a5a39ee7bf008864556f27870615b9, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:00:28,016 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731751217321.75a5a39ee7bf008864556f27870615b9., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=75a5a39ee7bf008864556f27870615b9, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:00:28,016 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731751217321.75a5a39ee7bf008864556f27870615b9., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 because the exception is null or not the one we care about 2024-11-16T10:00:28,016 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=75a5a39ee7bf008864556f27870615b9, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T10:00:28,016 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=75a5a39ee7bf008864556f27870615b9, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T10:00:28,016 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34428 deadline: 1731751288013, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=75a5a39ee7bf008864556f27870615b9, server=037c58a19ddb,37267,1731751077844 2024-11-16T10:00:28,016 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34412 deadline: 1731751288013, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=75a5a39ee7bf008864556f27870615b9, server=037c58a19ddb,37267,1731751077844 2024-11-16T10:00:28,016 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=75a5a39ee7bf008864556f27870615b9, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T10:00:28,016 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731751217321.75a5a39ee7bf008864556f27870615b9., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 , the old value is region=TestAcidGuarantees,,1731751217321.75a5a39ee7bf008864556f27870615b9., hostname=037c58a19ddb,37267,1731751077844, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=75a5a39ee7bf008864556f27870615b9, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:00:28,016 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731751217321.75a5a39ee7bf008864556f27870615b9., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=75a5a39ee7bf008864556f27870615b9, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:00:28,016 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731751217321.75a5a39ee7bf008864556f27870615b9., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 , the old value is region=TestAcidGuarantees,,1731751217321.75a5a39ee7bf008864556f27870615b9., hostname=037c58a19ddb,37267,1731751077844, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=75a5a39ee7bf008864556f27870615b9, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:00:28,016 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34430 deadline: 1731751288013, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=75a5a39ee7bf008864556f27870615b9, server=037c58a19ddb,37267,1731751077844 2024-11-16T10:00:28,016 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731751217321.75a5a39ee7bf008864556f27870615b9., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 because the exception is null or not the one we care about 2024-11-16T10:00:28,016 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731751217321.75a5a39ee7bf008864556f27870615b9., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=75a5a39ee7bf008864556f27870615b9, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:00:28,017 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731751217321.75a5a39ee7bf008864556f27870615b9., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 because the exception is null or not the one we care about 2024-11-16T10:00:28,017 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731751217321.75a5a39ee7bf008864556f27870615b9., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 , the old value is region=TestAcidGuarantees,,1731751217321.75a5a39ee7bf008864556f27870615b9., hostname=037c58a19ddb,37267,1731751077844, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=75a5a39ee7bf008864556f27870615b9, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:00:28,017 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731751217321.75a5a39ee7bf008864556f27870615b9., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=75a5a39ee7bf008864556f27870615b9, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:00:28,017 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731751217321.75a5a39ee7bf008864556f27870615b9., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 because the exception is null or not the one we care about 2024-11-16T10:00:28,139 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=37267 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=157 2024-11-16T10:00:28,139 DEBUG [RS_FLUSH_OPERATIONS-regionserver/037c58a19ddb:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731751217321.75a5a39ee7bf008864556f27870615b9. 2024-11-16T10:00:28,139 DEBUG [RS_FLUSH_OPERATIONS-regionserver/037c58a19ddb:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731751217321.75a5a39ee7bf008864556f27870615b9. as already flushing 2024-11-16T10:00:28,139 DEBUG [RS_FLUSH_OPERATIONS-regionserver/037c58a19ddb:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731751217321.75a5a39ee7bf008864556f27870615b9. 2024-11-16T10:00:28,139 ERROR [RS_FLUSH_OPERATIONS-regionserver/037c58a19ddb:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] handler.RSProcedureHandler(58): pid=157 java.io.IOException: Unable to complete flush {ENCODED => 75a5a39ee7bf008864556f27870615b9, NAME => 'TestAcidGuarantees,,1731751217321.75a5a39ee7bf008864556f27870615b9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T10:00:28,139 DEBUG [RS_FLUSH_OPERATIONS-regionserver/037c58a19ddb:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=157 java.io.IOException: Unable to complete flush {ENCODED => 75a5a39ee7bf008864556f27870615b9, NAME => 'TestAcidGuarantees,,1731751217321.75a5a39ee7bf008864556f27870615b9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T10:00:28,140 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37877 {}] master.HMaster(4177): Remote procedure failed, pid=157 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 75a5a39ee7bf008864556f27870615b9, NAME => 'TestAcidGuarantees,,1731751217321.75a5a39ee7bf008864556f27870615b9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 75a5a39ee7bf008864556f27870615b9, NAME => 'TestAcidGuarantees,,1731751217321.75a5a39ee7bf008864556f27870615b9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T10:00:28,203 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37877 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=156 2024-11-16T10:00:28,273 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=156, memsize=49.2 K, hasBloomFilter=true, into tmp file hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/75a5a39ee7bf008864556f27870615b9/.tmp/A/80e1e8f18c254519b6be286bcb29f3db 2024-11-16T10:00:28,279 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/75a5a39ee7bf008864556f27870615b9/.tmp/B/36b2600aa11f4987924ad638f9bf96a5 is 50, key is test_row_0/B:col10/1731751227246/Put/seqid=0 2024-11-16T10:00:28,282 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37591 is added to blk_1073742409_1585 (size=12151) 2024-11-16T10:00:28,284 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=156 (bloomFilter=true), to=hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/75a5a39ee7bf008864556f27870615b9/.tmp/B/36b2600aa11f4987924ad638f9bf96a5 2024-11-16T10:00:28,290 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/75a5a39ee7bf008864556f27870615b9/.tmp/C/38bbc22a1129485f90b4650be6aa0326 is 50, key is test_row_0/C:col10/1731751227246/Put/seqid=0 2024-11-16T10:00:28,292 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=37267 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=157 2024-11-16T10:00:28,292 DEBUG [RS_FLUSH_OPERATIONS-regionserver/037c58a19ddb:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731751217321.75a5a39ee7bf008864556f27870615b9. 2024-11-16T10:00:28,292 DEBUG [RS_FLUSH_OPERATIONS-regionserver/037c58a19ddb:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731751217321.75a5a39ee7bf008864556f27870615b9. as already flushing 2024-11-16T10:00:28,292 DEBUG [RS_FLUSH_OPERATIONS-regionserver/037c58a19ddb:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731751217321.75a5a39ee7bf008864556f27870615b9. 2024-11-16T10:00:28,292 ERROR [RS_FLUSH_OPERATIONS-regionserver/037c58a19ddb:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] handler.RSProcedureHandler(58): pid=157 java.io.IOException: Unable to complete flush {ENCODED => 75a5a39ee7bf008864556f27870615b9, NAME => 'TestAcidGuarantees,,1731751217321.75a5a39ee7bf008864556f27870615b9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T10:00:28,293 DEBUG [RS_FLUSH_OPERATIONS-regionserver/037c58a19ddb:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=157 java.io.IOException: Unable to complete flush {ENCODED => 75a5a39ee7bf008864556f27870615b9, NAME => 'TestAcidGuarantees,,1731751217321.75a5a39ee7bf008864556f27870615b9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T10:00:28,293 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37877 {}] master.HMaster(4177): Remote procedure failed, pid=157 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 75a5a39ee7bf008864556f27870615b9, NAME => 'TestAcidGuarantees,,1731751217321.75a5a39ee7bf008864556f27870615b9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 75a5a39ee7bf008864556f27870615b9, NAME => 'TestAcidGuarantees,,1731751217321.75a5a39ee7bf008864556f27870615b9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T10:00:28,294 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37591 is added to blk_1073742410_1586 (size=12151) 2024-11-16T10:00:28,445 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=37267 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=157 2024-11-16T10:00:28,446 DEBUG [RS_FLUSH_OPERATIONS-regionserver/037c58a19ddb:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731751217321.75a5a39ee7bf008864556f27870615b9. 2024-11-16T10:00:28,446 DEBUG [RS_FLUSH_OPERATIONS-regionserver/037c58a19ddb:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731751217321.75a5a39ee7bf008864556f27870615b9. as already flushing 2024-11-16T10:00:28,446 DEBUG [RS_FLUSH_OPERATIONS-regionserver/037c58a19ddb:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731751217321.75a5a39ee7bf008864556f27870615b9. 2024-11-16T10:00:28,446 ERROR [RS_FLUSH_OPERATIONS-regionserver/037c58a19ddb:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] handler.RSProcedureHandler(58): pid=157 java.io.IOException: Unable to complete flush {ENCODED => 75a5a39ee7bf008864556f27870615b9, NAME => 'TestAcidGuarantees,,1731751217321.75a5a39ee7bf008864556f27870615b9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T10:00:28,446 DEBUG [RS_FLUSH_OPERATIONS-regionserver/037c58a19ddb:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=157 java.io.IOException: Unable to complete flush {ENCODED => 75a5a39ee7bf008864556f27870615b9, NAME => 'TestAcidGuarantees,,1731751217321.75a5a39ee7bf008864556f27870615b9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T10:00:28,446 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37877 {}] master.HMaster(4177): Remote procedure failed, pid=157 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 75a5a39ee7bf008864556f27870615b9, NAME => 'TestAcidGuarantees,,1731751217321.75a5a39ee7bf008864556f27870615b9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 75a5a39ee7bf008864556f27870615b9, NAME => 'TestAcidGuarantees,,1731751217321.75a5a39ee7bf008864556f27870615b9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T10:00:28,524 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=75a5a39ee7bf008864556f27870615b9, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T10:00:28,525 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34454 deadline: 1731751288522, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=75a5a39ee7bf008864556f27870615b9, server=037c58a19ddb,37267,1731751077844 2024-11-16T10:00:28,525 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=75a5a39ee7bf008864556f27870615b9, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T10:00:28,525 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731751217321.75a5a39ee7bf008864556f27870615b9., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 , the old value is region=TestAcidGuarantees,,1731751217321.75a5a39ee7bf008864556f27870615b9., hostname=037c58a19ddb,37267,1731751077844, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=75a5a39ee7bf008864556f27870615b9, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:00:28,525 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731751217321.75a5a39ee7bf008864556f27870615b9., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=75a5a39ee7bf008864556f27870615b9, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:00:28,525 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34392 deadline: 1731751288522, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=75a5a39ee7bf008864556f27870615b9, server=037c58a19ddb,37267,1731751077844 2024-11-16T10:00:28,525 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731751217321.75a5a39ee7bf008864556f27870615b9., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 because the exception is null or not the one we care about 2024-11-16T10:00:28,525 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=75a5a39ee7bf008864556f27870615b9, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T10:00:28,525 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731751217321.75a5a39ee7bf008864556f27870615b9., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 , the old value is region=TestAcidGuarantees,,1731751217321.75a5a39ee7bf008864556f27870615b9., hostname=037c58a19ddb,37267,1731751077844, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=75a5a39ee7bf008864556f27870615b9, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:00:28,525 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34428 deadline: 1731751288523, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=75a5a39ee7bf008864556f27870615b9, server=037c58a19ddb,37267,1731751077844 2024-11-16T10:00:28,525 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731751217321.75a5a39ee7bf008864556f27870615b9., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=75a5a39ee7bf008864556f27870615b9, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:00:28,525 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731751217321.75a5a39ee7bf008864556f27870615b9., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 because the exception is null or not the one we care about 2024-11-16T10:00:28,525 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=75a5a39ee7bf008864556f27870615b9, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T10:00:28,525 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731751217321.75a5a39ee7bf008864556f27870615b9., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 , the old value is region=TestAcidGuarantees,,1731751217321.75a5a39ee7bf008864556f27870615b9., hostname=037c58a19ddb,37267,1731751077844, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=75a5a39ee7bf008864556f27870615b9, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:00:28,526 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731751217321.75a5a39ee7bf008864556f27870615b9., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=75a5a39ee7bf008864556f27870615b9, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:00:28,526 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34430 deadline: 1731751288523, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=75a5a39ee7bf008864556f27870615b9, server=037c58a19ddb,37267,1731751077844 2024-11-16T10:00:28,526 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731751217321.75a5a39ee7bf008864556f27870615b9., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 because the exception is null or not the one we care about 2024-11-16T10:00:28,526 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=75a5a39ee7bf008864556f27870615b9, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T10:00:28,526 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731751217321.75a5a39ee7bf008864556f27870615b9., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 , the old value is region=TestAcidGuarantees,,1731751217321.75a5a39ee7bf008864556f27870615b9., hostname=037c58a19ddb,37267,1731751077844, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=75a5a39ee7bf008864556f27870615b9, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:00:28,526 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34412 deadline: 1731751288523, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=75a5a39ee7bf008864556f27870615b9, server=037c58a19ddb,37267,1731751077844 2024-11-16T10:00:28,526 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731751217321.75a5a39ee7bf008864556f27870615b9., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=75a5a39ee7bf008864556f27870615b9, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:00:28,526 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731751217321.75a5a39ee7bf008864556f27870615b9., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 because the exception is null or not the one we care about 2024-11-16T10:00:28,526 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731751217321.75a5a39ee7bf008864556f27870615b9., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 , the old value is region=TestAcidGuarantees,,1731751217321.75a5a39ee7bf008864556f27870615b9., hostname=037c58a19ddb,37267,1731751077844, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=75a5a39ee7bf008864556f27870615b9, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:00:28,526 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731751217321.75a5a39ee7bf008864556f27870615b9., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=75a5a39ee7bf008864556f27870615b9, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:00:28,526 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731751217321.75a5a39ee7bf008864556f27870615b9., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 because the exception is null or not the one we care about 2024-11-16T10:00:28,598 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=37267 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=157 2024-11-16T10:00:28,598 DEBUG [RS_FLUSH_OPERATIONS-regionserver/037c58a19ddb:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731751217321.75a5a39ee7bf008864556f27870615b9. 2024-11-16T10:00:28,598 DEBUG [RS_FLUSH_OPERATIONS-regionserver/037c58a19ddb:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731751217321.75a5a39ee7bf008864556f27870615b9. as already flushing 2024-11-16T10:00:28,598 DEBUG [RS_FLUSH_OPERATIONS-regionserver/037c58a19ddb:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731751217321.75a5a39ee7bf008864556f27870615b9. 2024-11-16T10:00:28,598 ERROR [RS_FLUSH_OPERATIONS-regionserver/037c58a19ddb:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] handler.RSProcedureHandler(58): pid=157 java.io.IOException: Unable to complete flush {ENCODED => 75a5a39ee7bf008864556f27870615b9, NAME => 'TestAcidGuarantees,,1731751217321.75a5a39ee7bf008864556f27870615b9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T10:00:28,599 DEBUG [RS_FLUSH_OPERATIONS-regionserver/037c58a19ddb:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=157 java.io.IOException: Unable to complete flush {ENCODED => 75a5a39ee7bf008864556f27870615b9, NAME => 'TestAcidGuarantees,,1731751217321.75a5a39ee7bf008864556f27870615b9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T10:00:28,599 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37877 {}] master.HMaster(4177): Remote procedure failed, pid=157 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 75a5a39ee7bf008864556f27870615b9, NAME => 'TestAcidGuarantees,,1731751217321.75a5a39ee7bf008864556f27870615b9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 75a5a39ee7bf008864556f27870615b9, NAME => 'TestAcidGuarantees,,1731751217321.75a5a39ee7bf008864556f27870615b9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T10:00:28,695 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=156 (bloomFilter=true), to=hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/75a5a39ee7bf008864556f27870615b9/.tmp/C/38bbc22a1129485f90b4650be6aa0326 2024-11-16T10:00:28,698 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/75a5a39ee7bf008864556f27870615b9/.tmp/A/80e1e8f18c254519b6be286bcb29f3db as hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/75a5a39ee7bf008864556f27870615b9/A/80e1e8f18c254519b6be286bcb29f3db 2024-11-16T10:00:28,702 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/75a5a39ee7bf008864556f27870615b9/A/80e1e8f18c254519b6be286bcb29f3db, entries=200, sequenceid=156, filesize=38.8 K 2024-11-16T10:00:28,702 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/75a5a39ee7bf008864556f27870615b9/.tmp/B/36b2600aa11f4987924ad638f9bf96a5 as hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/75a5a39ee7bf008864556f27870615b9/B/36b2600aa11f4987924ad638f9bf96a5 2024-11-16T10:00:28,705 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/75a5a39ee7bf008864556f27870615b9/B/36b2600aa11f4987924ad638f9bf96a5, entries=150, sequenceid=156, filesize=11.9 K 2024-11-16T10:00:28,706 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/75a5a39ee7bf008864556f27870615b9/.tmp/C/38bbc22a1129485f90b4650be6aa0326 as hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/75a5a39ee7bf008864556f27870615b9/C/38bbc22a1129485f90b4650be6aa0326 2024-11-16T10:00:28,710 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/75a5a39ee7bf008864556f27870615b9/C/38bbc22a1129485f90b4650be6aa0326, entries=150, sequenceid=156, filesize=11.9 K 2024-11-16T10:00:28,711 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~147.60 KB/151140, heapSize ~387.42 KB/396720, currentSize=53.67 KB/54960 for 75a5a39ee7bf008864556f27870615b9 in 1343ms, sequenceid=156, compaction requested=false 2024-11-16T10:00:28,711 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 75a5a39ee7bf008864556f27870615b9: 2024-11-16T10:00:28,751 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=37267 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=157 2024-11-16T10:00:28,751 DEBUG [RS_FLUSH_OPERATIONS-regionserver/037c58a19ddb:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731751217321.75a5a39ee7bf008864556f27870615b9. 2024-11-16T10:00:28,751 INFO [RS_FLUSH_OPERATIONS-regionserver/037c58a19ddb:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HRegion(2902): Flushing 75a5a39ee7bf008864556f27870615b9 3/3 column families, dataSize=53.67 KB heapSize=141.38 KB 2024-11-16T10:00:28,757 DEBUG [RS_FLUSH_OPERATIONS-regionserver/037c58a19ddb:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241116287ae30978a84543903d26b650bf83c2_75a5a39ee7bf008864556f27870615b9 is 50, key is test_row_0/A:col10/1731751227382/Put/seqid=0 2024-11-16T10:00:28,760 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37591 is added to blk_1073742411_1587 (size=12304) 2024-11-16T10:00:29,161 DEBUG [RS_FLUSH_OPERATIONS-regionserver/037c58a19ddb:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:29,165 INFO [RS_FLUSH_OPERATIONS-regionserver/037c58a19ddb:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241116287ae30978a84543903d26b650bf83c2_75a5a39ee7bf008864556f27870615b9 to hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241116287ae30978a84543903d26b650bf83c2_75a5a39ee7bf008864556f27870615b9 2024-11-16T10:00:29,166 DEBUG [RS_FLUSH_OPERATIONS-regionserver/037c58a19ddb:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/75a5a39ee7bf008864556f27870615b9/.tmp/A/5ba81c6f8c6544a29da2e79b2c8b49d0, store: [table=TestAcidGuarantees family=A region=75a5a39ee7bf008864556f27870615b9] 2024-11-16T10:00:29,166 DEBUG [RS_FLUSH_OPERATIONS-regionserver/037c58a19ddb:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/75a5a39ee7bf008864556f27870615b9/.tmp/A/5ba81c6f8c6544a29da2e79b2c8b49d0 is 175, key is test_row_0/A:col10/1731751227382/Put/seqid=0 2024-11-16T10:00:29,169 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37591 is added to blk_1073742412_1588 (size=31105) 2024-11-16T10:00:29,172 INFO [RS_FLUSH_OPERATIONS-regionserver/037c58a19ddb:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=170, memsize=17.9 K, hasBloomFilter=true, into tmp file hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/75a5a39ee7bf008864556f27870615b9/.tmp/A/5ba81c6f8c6544a29da2e79b2c8b49d0 2024-11-16T10:00:29,179 DEBUG [RS_FLUSH_OPERATIONS-regionserver/037c58a19ddb:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/75a5a39ee7bf008864556f27870615b9/.tmp/B/0cef038cd17c43eab448fd42a00414aa is 50, key is test_row_0/B:col10/1731751227382/Put/seqid=0 2024-11-16T10:00:29,183 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37591 is added to blk_1073742413_1589 (size=12151) 2024-11-16T10:00:29,184 INFO [RS_FLUSH_OPERATIONS-regionserver/037c58a19ddb:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=170 (bloomFilter=true), to=hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/75a5a39ee7bf008864556f27870615b9/.tmp/B/0cef038cd17c43eab448fd42a00414aa 2024-11-16T10:00:29,189 DEBUG [RS_FLUSH_OPERATIONS-regionserver/037c58a19ddb:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/75a5a39ee7bf008864556f27870615b9/.tmp/C/eece82162ad545eea9eedfee8e3109fc is 50, key is test_row_0/C:col10/1731751227382/Put/seqid=0 2024-11-16T10:00:29,193 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37591 is added to blk_1073742414_1590 (size=12151) 2024-11-16T10:00:29,213 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37877 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=156 2024-11-16T10:00:29,536 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] regionserver.HRegion(8855): Flush requested on 75a5a39ee7bf008864556f27870615b9 2024-11-16T10:00:29,536 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731751217321.75a5a39ee7bf008864556f27870615b9. as already flushing 2024-11-16T10:00:29,590 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=75a5a39ee7bf008864556f27870615b9, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T10:00:29,590 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34412 deadline: 1731751289583, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=75a5a39ee7bf008864556f27870615b9, server=037c58a19ddb,37267,1731751077844 2024-11-16T10:00:29,590 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=75a5a39ee7bf008864556f27870615b9, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T10:00:29,590 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731751217321.75a5a39ee7bf008864556f27870615b9., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 , the old value is region=TestAcidGuarantees,,1731751217321.75a5a39ee7bf008864556f27870615b9., hostname=037c58a19ddb,37267,1731751077844, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=75a5a39ee7bf008864556f27870615b9, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:00:29,590 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731751217321.75a5a39ee7bf008864556f27870615b9., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=75a5a39ee7bf008864556f27870615b9, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:00:29,590 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34430 deadline: 1731751289583, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=75a5a39ee7bf008864556f27870615b9, server=037c58a19ddb,37267,1731751077844 2024-11-16T10:00:29,590 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731751217321.75a5a39ee7bf008864556f27870615b9., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 because the exception is null or not the one we care about 2024-11-16T10:00:29,590 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=75a5a39ee7bf008864556f27870615b9, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T10:00:29,591 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34392 deadline: 1731751289587, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=75a5a39ee7bf008864556f27870615b9, server=037c58a19ddb,37267,1731751077844 2024-11-16T10:00:29,591 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731751217321.75a5a39ee7bf008864556f27870615b9., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 , the old value is region=TestAcidGuarantees,,1731751217321.75a5a39ee7bf008864556f27870615b9., hostname=037c58a19ddb,37267,1731751077844, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=75a5a39ee7bf008864556f27870615b9, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:00:29,591 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731751217321.75a5a39ee7bf008864556f27870615b9., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=75a5a39ee7bf008864556f27870615b9, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:00:29,591 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731751217321.75a5a39ee7bf008864556f27870615b9., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 because the exception is null or not the one we care about 2024-11-16T10:00:29,591 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731751217321.75a5a39ee7bf008864556f27870615b9., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 , the old value is region=TestAcidGuarantees,,1731751217321.75a5a39ee7bf008864556f27870615b9., hostname=037c58a19ddb,37267,1731751077844, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=75a5a39ee7bf008864556f27870615b9, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:00:29,591 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731751217321.75a5a39ee7bf008864556f27870615b9., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=75a5a39ee7bf008864556f27870615b9, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:00:29,591 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731751217321.75a5a39ee7bf008864556f27870615b9., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 because the exception is null or not the one we care about 2024-11-16T10:00:29,594 INFO [RS_FLUSH_OPERATIONS-regionserver/037c58a19ddb:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=170 (bloomFilter=true), to=hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/75a5a39ee7bf008864556f27870615b9/.tmp/C/eece82162ad545eea9eedfee8e3109fc 2024-11-16T10:00:29,594 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=75a5a39ee7bf008864556f27870615b9, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T10:00:29,594 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34428 deadline: 1731751289589, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=75a5a39ee7bf008864556f27870615b9, server=037c58a19ddb,37267,1731751077844 2024-11-16T10:00:29,594 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=75a5a39ee7bf008864556f27870615b9, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T10:00:29,594 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731751217321.75a5a39ee7bf008864556f27870615b9., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 , the old value is region=TestAcidGuarantees,,1731751217321.75a5a39ee7bf008864556f27870615b9., hostname=037c58a19ddb,37267,1731751077844, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=75a5a39ee7bf008864556f27870615b9, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:00:29,595 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731751217321.75a5a39ee7bf008864556f27870615b9., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=75a5a39ee7bf008864556f27870615b9, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:00:29,595 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34454 deadline: 1731751289590, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=75a5a39ee7bf008864556f27870615b9, server=037c58a19ddb,37267,1731751077844 2024-11-16T10:00:29,595 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731751217321.75a5a39ee7bf008864556f27870615b9., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 because the exception is null or not the one we care about 2024-11-16T10:00:29,595 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731751217321.75a5a39ee7bf008864556f27870615b9., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 , the old value is region=TestAcidGuarantees,,1731751217321.75a5a39ee7bf008864556f27870615b9., hostname=037c58a19ddb,37267,1731751077844, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=75a5a39ee7bf008864556f27870615b9, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:00:29,595 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731751217321.75a5a39ee7bf008864556f27870615b9., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=75a5a39ee7bf008864556f27870615b9, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:00:29,595 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731751217321.75a5a39ee7bf008864556f27870615b9., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 because the exception is null or not the one we care about 2024-11-16T10:00:29,598 DEBUG [RS_FLUSH_OPERATIONS-regionserver/037c58a19ddb:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/75a5a39ee7bf008864556f27870615b9/.tmp/A/5ba81c6f8c6544a29da2e79b2c8b49d0 as hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/75a5a39ee7bf008864556f27870615b9/A/5ba81c6f8c6544a29da2e79b2c8b49d0 2024-11-16T10:00:29,601 INFO [RS_FLUSH_OPERATIONS-regionserver/037c58a19ddb:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/75a5a39ee7bf008864556f27870615b9/A/5ba81c6f8c6544a29da2e79b2c8b49d0, entries=150, sequenceid=170, filesize=30.4 K 2024-11-16T10:00:29,602 DEBUG [RS_FLUSH_OPERATIONS-regionserver/037c58a19ddb:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/75a5a39ee7bf008864556f27870615b9/.tmp/B/0cef038cd17c43eab448fd42a00414aa as hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/75a5a39ee7bf008864556f27870615b9/B/0cef038cd17c43eab448fd42a00414aa 2024-11-16T10:00:29,606 INFO [RS_FLUSH_OPERATIONS-regionserver/037c58a19ddb:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/75a5a39ee7bf008864556f27870615b9/B/0cef038cd17c43eab448fd42a00414aa, entries=150, sequenceid=170, filesize=11.9 K 2024-11-16T10:00:29,606 DEBUG [RS_FLUSH_OPERATIONS-regionserver/037c58a19ddb:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/75a5a39ee7bf008864556f27870615b9/.tmp/C/eece82162ad545eea9eedfee8e3109fc as hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/75a5a39ee7bf008864556f27870615b9/C/eece82162ad545eea9eedfee8e3109fc 2024-11-16T10:00:29,615 INFO [RS_FLUSH_OPERATIONS-regionserver/037c58a19ddb:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/75a5a39ee7bf008864556f27870615b9/C/eece82162ad545eea9eedfee8e3109fc, entries=150, sequenceid=170, filesize=11.9 K 2024-11-16T10:00:29,616 INFO [RS_FLUSH_OPERATIONS-regionserver/037c58a19ddb:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~141.33 KB/144720, currentSize=147.60 KB/151140 for 75a5a39ee7bf008864556f27870615b9 in 864ms, sequenceid=170, compaction requested=true 2024-11-16T10:00:29,616 DEBUG [RS_FLUSH_OPERATIONS-regionserver/037c58a19ddb:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HRegion(2603): Flush status journal for 75a5a39ee7bf008864556f27870615b9: 2024-11-16T10:00:29,616 DEBUG [RS_FLUSH_OPERATIONS-regionserver/037c58a19ddb:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731751217321.75a5a39ee7bf008864556f27870615b9. 2024-11-16T10:00:29,616 DEBUG [RS_FLUSH_OPERATIONS-regionserver/037c58a19ddb:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=157 2024-11-16T10:00:29,616 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37877 {}] master.HMaster(4169): Remote procedure done, pid=157 2024-11-16T10:00:29,620 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=157, resume processing ppid=156 2024-11-16T10:00:29,620 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=157, ppid=156, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.5500 sec 2024-11-16T10:00:29,622 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=156, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=156, table=TestAcidGuarantees in 2.5560 sec 2024-11-16T10:00:29,696 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] regionserver.HRegion(8855): Flush requested on 75a5a39ee7bf008864556f27870615b9 2024-11-16T10:00:29,696 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-16T10:00:29,696 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 75a5a39ee7bf008864556f27870615b9 3/3 column families, dataSize=154.31 KB heapSize=405.05 KB 2024-11-16T10:00:29,704 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111620c9d856242848abb88eb70393008744_75a5a39ee7bf008864556f27870615b9 is 50, key is test_row_0/A:col10/1731751229586/Put/seqid=0 2024-11-16T10:00:29,708 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37591 is added to blk_1073742415_1591 (size=14794) 2024-11-16T10:00:29,708 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:29,711 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=75a5a39ee7bf008864556f27870615b9, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T10:00:29,711 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34412 deadline: 1731751289705, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=75a5a39ee7bf008864556f27870615b9, server=037c58a19ddb,37267,1731751077844 2024-11-16T10:00:29,711 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=75a5a39ee7bf008864556f27870615b9, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T10:00:29,711 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34430 deadline: 1731751289706, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=75a5a39ee7bf008864556f27870615b9, server=037c58a19ddb,37267,1731751077844 2024-11-16T10:00:29,711 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=75a5a39ee7bf008864556f27870615b9, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T10:00:29,711 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34392 deadline: 1731751289706, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=75a5a39ee7bf008864556f27870615b9, server=037c58a19ddb,37267,1731751077844 2024-11-16T10:00:29,712 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=75a5a39ee7bf008864556f27870615b9, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T10:00:29,712 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34428 deadline: 1731751289707, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=75a5a39ee7bf008864556f27870615b9, server=037c58a19ddb,37267,1731751077844 2024-11-16T10:00:29,712 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731751217321.75a5a39ee7bf008864556f27870615b9., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 , the old value is region=TestAcidGuarantees,,1731751217321.75a5a39ee7bf008864556f27870615b9., hostname=037c58a19ddb,37267,1731751077844, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=75a5a39ee7bf008864556f27870615b9, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:00:29,712 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731751217321.75a5a39ee7bf008864556f27870615b9., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 , the old value is region=TestAcidGuarantees,,1731751217321.75a5a39ee7bf008864556f27870615b9., hostname=037c58a19ddb,37267,1731751077844, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=75a5a39ee7bf008864556f27870615b9, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:00:29,712 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731751217321.75a5a39ee7bf008864556f27870615b9., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=75a5a39ee7bf008864556f27870615b9, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:00:29,712 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731751217321.75a5a39ee7bf008864556f27870615b9., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=75a5a39ee7bf008864556f27870615b9, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:00:29,712 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731751217321.75a5a39ee7bf008864556f27870615b9., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 because the exception is null or not the one we care about 2024-11-16T10:00:29,712 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731751217321.75a5a39ee7bf008864556f27870615b9., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 because the exception is null or not the one we care about 2024-11-16T10:00:29,712 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111620c9d856242848abb88eb70393008744_75a5a39ee7bf008864556f27870615b9 to hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111620c9d856242848abb88eb70393008744_75a5a39ee7bf008864556f27870615b9 2024-11-16T10:00:29,712 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=75a5a39ee7bf008864556f27870615b9, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T10:00:29,712 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34454 deadline: 1731751289707, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=75a5a39ee7bf008864556f27870615b9, server=037c58a19ddb,37267,1731751077844 2024-11-16T10:00:29,712 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731751217321.75a5a39ee7bf008864556f27870615b9., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 , the old value is region=TestAcidGuarantees,,1731751217321.75a5a39ee7bf008864556f27870615b9., hostname=037c58a19ddb,37267,1731751077844, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=75a5a39ee7bf008864556f27870615b9, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:00:29,712 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731751217321.75a5a39ee7bf008864556f27870615b9., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=75a5a39ee7bf008864556f27870615b9, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:00:29,712 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731751217321.75a5a39ee7bf008864556f27870615b9., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 because the exception is null or not the one we care about 2024-11-16T10:00:29,712 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731751217321.75a5a39ee7bf008864556f27870615b9., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 , the old value is region=TestAcidGuarantees,,1731751217321.75a5a39ee7bf008864556f27870615b9., hostname=037c58a19ddb,37267,1731751077844, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=75a5a39ee7bf008864556f27870615b9, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:00:29,712 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731751217321.75a5a39ee7bf008864556f27870615b9., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=75a5a39ee7bf008864556f27870615b9, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:00:29,713 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731751217321.75a5a39ee7bf008864556f27870615b9., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 because the exception is null or not the one we care about 2024-11-16T10:00:29,713 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731751217321.75a5a39ee7bf008864556f27870615b9., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 , the old value is region=TestAcidGuarantees,,1731751217321.75a5a39ee7bf008864556f27870615b9., hostname=037c58a19ddb,37267,1731751077844, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=75a5a39ee7bf008864556f27870615b9, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:00:29,713 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731751217321.75a5a39ee7bf008864556f27870615b9., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=75a5a39ee7bf008864556f27870615b9, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:00:29,713 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731751217321.75a5a39ee7bf008864556f27870615b9., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 because the exception is null or not the one we care about 2024-11-16T10:00:29,713 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/75a5a39ee7bf008864556f27870615b9/.tmp/A/955ac682fed54cc0af1fa49e6c502dd2, store: [table=TestAcidGuarantees family=A region=75a5a39ee7bf008864556f27870615b9] 2024-11-16T10:00:29,714 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/75a5a39ee7bf008864556f27870615b9/.tmp/A/955ac682fed54cc0af1fa49e6c502dd2 is 175, key is test_row_0/A:col10/1731751229586/Put/seqid=0 2024-11-16T10:00:29,718 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37591 is added to blk_1073742416_1592 (size=39749) 2024-11-16T10:00:29,826 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=75a5a39ee7bf008864556f27870615b9, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T10:00:29,826 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34412 deadline: 1731751289823, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=75a5a39ee7bf008864556f27870615b9, server=037c58a19ddb,37267,1731751077844 2024-11-16T10:00:29,827 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=75a5a39ee7bf008864556f27870615b9, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T10:00:29,827 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34430 deadline: 1731751289823, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=75a5a39ee7bf008864556f27870615b9, server=037c58a19ddb,37267,1731751077844 2024-11-16T10:00:29,827 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731751217321.75a5a39ee7bf008864556f27870615b9., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 , the old value is region=TestAcidGuarantees,,1731751217321.75a5a39ee7bf008864556f27870615b9., hostname=037c58a19ddb,37267,1731751077844, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=75a5a39ee7bf008864556f27870615b9, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:00:29,827 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731751217321.75a5a39ee7bf008864556f27870615b9., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=75a5a39ee7bf008864556f27870615b9, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:00:29,827 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731751217321.75a5a39ee7bf008864556f27870615b9., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 because the exception is null or not the one we care about 2024-11-16T10:00:29,827 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731751217321.75a5a39ee7bf008864556f27870615b9., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 , the old value is region=TestAcidGuarantees,,1731751217321.75a5a39ee7bf008864556f27870615b9., hostname=037c58a19ddb,37267,1731751077844, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=75a5a39ee7bf008864556f27870615b9, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:00:29,827 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=75a5a39ee7bf008864556f27870615b9, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T10:00:29,827 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731751217321.75a5a39ee7bf008864556f27870615b9., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=75a5a39ee7bf008864556f27870615b9, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:00:29,827 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731751217321.75a5a39ee7bf008864556f27870615b9., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 because the exception is null or not the one we care about 2024-11-16T10:00:29,827 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34392 deadline: 1731751289823, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=75a5a39ee7bf008864556f27870615b9, server=037c58a19ddb,37267,1731751077844 2024-11-16T10:00:29,827 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731751217321.75a5a39ee7bf008864556f27870615b9., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 , the old value is region=TestAcidGuarantees,,1731751217321.75a5a39ee7bf008864556f27870615b9., hostname=037c58a19ddb,37267,1731751077844, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=75a5a39ee7bf008864556f27870615b9, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:00:29,827 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=75a5a39ee7bf008864556f27870615b9, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T10:00:29,827 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731751217321.75a5a39ee7bf008864556f27870615b9., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=75a5a39ee7bf008864556f27870615b9, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:00:29,827 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731751217321.75a5a39ee7bf008864556f27870615b9., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 because the exception is null or not the one we care about 2024-11-16T10:00:29,827 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34428 deadline: 1731751289823, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=75a5a39ee7bf008864556f27870615b9, server=037c58a19ddb,37267,1731751077844 2024-11-16T10:00:29,827 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731751217321.75a5a39ee7bf008864556f27870615b9., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 , the old value is region=TestAcidGuarantees,,1731751217321.75a5a39ee7bf008864556f27870615b9., hostname=037c58a19ddb,37267,1731751077844, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=75a5a39ee7bf008864556f27870615b9, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:00:29,828 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731751217321.75a5a39ee7bf008864556f27870615b9., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=75a5a39ee7bf008864556f27870615b9, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:00:29,828 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731751217321.75a5a39ee7bf008864556f27870615b9., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 because the exception is null or not the one we care about 2024-11-16T10:00:29,827 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=75a5a39ee7bf008864556f27870615b9, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T10:00:29,828 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34454 deadline: 1731751289824, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=75a5a39ee7bf008864556f27870615b9, server=037c58a19ddb,37267,1731751077844 2024-11-16T10:00:29,828 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731751217321.75a5a39ee7bf008864556f27870615b9., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 , the old value is region=TestAcidGuarantees,,1731751217321.75a5a39ee7bf008864556f27870615b9., hostname=037c58a19ddb,37267,1731751077844, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=75a5a39ee7bf008864556f27870615b9, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:00:29,828 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731751217321.75a5a39ee7bf008864556f27870615b9., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=75a5a39ee7bf008864556f27870615b9, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:00:29,828 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731751217321.75a5a39ee7bf008864556f27870615b9., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 because the exception is null or not the one we care about 2024-11-16T10:00:30,034 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=75a5a39ee7bf008864556f27870615b9, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T10:00:30,034 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34412 deadline: 1731751290032, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=75a5a39ee7bf008864556f27870615b9, server=037c58a19ddb,37267,1731751077844 2024-11-16T10:00:30,034 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=75a5a39ee7bf008864556f27870615b9, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T10:00:30,034 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34430 deadline: 1731751290032, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=75a5a39ee7bf008864556f27870615b9, server=037c58a19ddb,37267,1731751077844 2024-11-16T10:00:30,034 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731751217321.75a5a39ee7bf008864556f27870615b9., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 , the old value is region=TestAcidGuarantees,,1731751217321.75a5a39ee7bf008864556f27870615b9., hostname=037c58a19ddb,37267,1731751077844, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=75a5a39ee7bf008864556f27870615b9, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:00:30,034 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731751217321.75a5a39ee7bf008864556f27870615b9., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=75a5a39ee7bf008864556f27870615b9, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:00:30,034 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731751217321.75a5a39ee7bf008864556f27870615b9., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 because the exception is null or not the one we care about 2024-11-16T10:00:30,034 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=75a5a39ee7bf008864556f27870615b9, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T10:00:30,035 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731751217321.75a5a39ee7bf008864556f27870615b9., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 , the old value is region=TestAcidGuarantees,,1731751217321.75a5a39ee7bf008864556f27870615b9., hostname=037c58a19ddb,37267,1731751077844, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=75a5a39ee7bf008864556f27870615b9, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:00:30,035 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731751217321.75a5a39ee7bf008864556f27870615b9., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=75a5a39ee7bf008864556f27870615b9, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:00:30,035 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34392 deadline: 1731751290032, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=75a5a39ee7bf008864556f27870615b9, server=037c58a19ddb,37267,1731751077844 2024-11-16T10:00:30,035 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731751217321.75a5a39ee7bf008864556f27870615b9., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 because the exception is null or not the one we care about 2024-11-16T10:00:30,035 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731751217321.75a5a39ee7bf008864556f27870615b9., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 , the old value is region=TestAcidGuarantees,,1731751217321.75a5a39ee7bf008864556f27870615b9., hostname=037c58a19ddb,37267,1731751077844, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=75a5a39ee7bf008864556f27870615b9, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:00:30,035 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=75a5a39ee7bf008864556f27870615b9, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T10:00:30,035 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731751217321.75a5a39ee7bf008864556f27870615b9., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=75a5a39ee7bf008864556f27870615b9, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:00:30,035 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731751217321.75a5a39ee7bf008864556f27870615b9., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 because the exception is null or not the one we care about 2024-11-16T10:00:30,035 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34428 deadline: 1731751290032, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=75a5a39ee7bf008864556f27870615b9, server=037c58a19ddb,37267,1731751077844 2024-11-16T10:00:30,035 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731751217321.75a5a39ee7bf008864556f27870615b9., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 , the old value is region=TestAcidGuarantees,,1731751217321.75a5a39ee7bf008864556f27870615b9., hostname=037c58a19ddb,37267,1731751077844, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=75a5a39ee7bf008864556f27870615b9, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:00:30,035 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=75a5a39ee7bf008864556f27870615b9, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T10:00:30,035 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731751217321.75a5a39ee7bf008864556f27870615b9., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=75a5a39ee7bf008864556f27870615b9, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:00:30,035 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731751217321.75a5a39ee7bf008864556f27870615b9., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 because the exception is null or not the one we care about 2024-11-16T10:00:30,035 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34454 deadline: 1731751290033, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=75a5a39ee7bf008864556f27870615b9, server=037c58a19ddb,37267,1731751077844 2024-11-16T10:00:30,035 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731751217321.75a5a39ee7bf008864556f27870615b9., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 , the old value is region=TestAcidGuarantees,,1731751217321.75a5a39ee7bf008864556f27870615b9., hostname=037c58a19ddb,37267,1731751077844, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=75a5a39ee7bf008864556f27870615b9, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:00:30,035 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731751217321.75a5a39ee7bf008864556f27870615b9., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=75a5a39ee7bf008864556f27870615b9, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:00:30,035 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731751217321.75a5a39ee7bf008864556f27870615b9., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 because the exception is null or not the one we care about 2024-11-16T10:00:30,118 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=197, memsize=53.7 K, hasBloomFilter=true, into tmp file hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/75a5a39ee7bf008864556f27870615b9/.tmp/A/955ac682fed54cc0af1fa49e6c502dd2 2024-11-16T10:00:30,125 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/75a5a39ee7bf008864556f27870615b9/.tmp/B/eb93c6fdfe03475481ef8e2ff93de934 is 50, key is test_row_0/B:col10/1731751229586/Put/seqid=0 2024-11-16T10:00:30,132 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37591 is added to blk_1073742417_1593 (size=12151) 2024-11-16T10:00:30,345 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=75a5a39ee7bf008864556f27870615b9, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T10:00:30,346 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34412 deadline: 1731751290343, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=75a5a39ee7bf008864556f27870615b9, server=037c58a19ddb,37267,1731751077844 2024-11-16T10:00:30,346 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=75a5a39ee7bf008864556f27870615b9, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T10:00:30,346 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731751217321.75a5a39ee7bf008864556f27870615b9., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 , the old value is region=TestAcidGuarantees,,1731751217321.75a5a39ee7bf008864556f27870615b9., hostname=037c58a19ddb,37267,1731751077844, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=75a5a39ee7bf008864556f27870615b9, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:00:30,346 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34430 deadline: 1731751290343, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=75a5a39ee7bf008864556f27870615b9, server=037c58a19ddb,37267,1731751077844 2024-11-16T10:00:30,346 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731751217321.75a5a39ee7bf008864556f27870615b9., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=75a5a39ee7bf008864556f27870615b9, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:00:30,346 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731751217321.75a5a39ee7bf008864556f27870615b9., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 because the exception is null or not the one we care about 2024-11-16T10:00:30,346 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=75a5a39ee7bf008864556f27870615b9, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T10:00:30,346 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731751217321.75a5a39ee7bf008864556f27870615b9., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 , the old value is region=TestAcidGuarantees,,1731751217321.75a5a39ee7bf008864556f27870615b9., hostname=037c58a19ddb,37267,1731751077844, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=75a5a39ee7bf008864556f27870615b9, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:00:30,346 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34392 deadline: 1731751290343, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=75a5a39ee7bf008864556f27870615b9, server=037c58a19ddb,37267,1731751077844 2024-11-16T10:00:30,346 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731751217321.75a5a39ee7bf008864556f27870615b9., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=75a5a39ee7bf008864556f27870615b9, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:00:30,346 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731751217321.75a5a39ee7bf008864556f27870615b9., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 because the exception is null or not the one we care about 2024-11-16T10:00:30,346 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731751217321.75a5a39ee7bf008864556f27870615b9., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 , the old value is region=TestAcidGuarantees,,1731751217321.75a5a39ee7bf008864556f27870615b9., hostname=037c58a19ddb,37267,1731751077844, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=75a5a39ee7bf008864556f27870615b9, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:00:30,346 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=75a5a39ee7bf008864556f27870615b9, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T10:00:30,346 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731751217321.75a5a39ee7bf008864556f27870615b9., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=75a5a39ee7bf008864556f27870615b9, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:00:30,346 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731751217321.75a5a39ee7bf008864556f27870615b9., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 because the exception is null or not the one we care about 2024-11-16T10:00:30,346 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34428 deadline: 1731751290343, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=75a5a39ee7bf008864556f27870615b9, server=037c58a19ddb,37267,1731751077844 2024-11-16T10:00:30,347 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=75a5a39ee7bf008864556f27870615b9, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T10:00:30,347 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731751217321.75a5a39ee7bf008864556f27870615b9., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 , the old value is region=TestAcidGuarantees,,1731751217321.75a5a39ee7bf008864556f27870615b9., hostname=037c58a19ddb,37267,1731751077844, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=75a5a39ee7bf008864556f27870615b9, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:00:30,347 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731751217321.75a5a39ee7bf008864556f27870615b9., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=75a5a39ee7bf008864556f27870615b9, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:00:30,347 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34454 deadline: 1731751290343, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=75a5a39ee7bf008864556f27870615b9, server=037c58a19ddb,37267,1731751077844 2024-11-16T10:00:30,347 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731751217321.75a5a39ee7bf008864556f27870615b9., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 because the exception is null or not the one we care about 2024-11-16T10:00:30,347 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731751217321.75a5a39ee7bf008864556f27870615b9., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 , the old value is region=TestAcidGuarantees,,1731751217321.75a5a39ee7bf008864556f27870615b9., hostname=037c58a19ddb,37267,1731751077844, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=75a5a39ee7bf008864556f27870615b9, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:00:30,347 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731751217321.75a5a39ee7bf008864556f27870615b9., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=75a5a39ee7bf008864556f27870615b9, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:00:30,347 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731751217321.75a5a39ee7bf008864556f27870615b9., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 because the exception is null or not the one we care about 2024-11-16T10:00:30,529 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=53.67 KB at sequenceid=197 (bloomFilter=true), to=hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/75a5a39ee7bf008864556f27870615b9/.tmp/B/eb93c6fdfe03475481ef8e2ff93de934 2024-11-16T10:00:30,535 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/75a5a39ee7bf008864556f27870615b9/.tmp/C/9c5660694a7a480caf5c5fc83c189dc4 is 50, key is test_row_0/C:col10/1731751229586/Put/seqid=0 2024-11-16T10:00:30,538 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37591 is added to blk_1073742418_1594 (size=12151) 2024-11-16T10:00:30,855 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=75a5a39ee7bf008864556f27870615b9, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T10:00:30,855 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34412 deadline: 1731751290853, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=75a5a39ee7bf008864556f27870615b9, server=037c58a19ddb,37267,1731751077844 2024-11-16T10:00:30,855 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=75a5a39ee7bf008864556f27870615b9, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T10:00:30,855 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731751217321.75a5a39ee7bf008864556f27870615b9., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 , the old value is region=TestAcidGuarantees,,1731751217321.75a5a39ee7bf008864556f27870615b9., hostname=037c58a19ddb,37267,1731751077844, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=75a5a39ee7bf008864556f27870615b9, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:00:30,855 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731751217321.75a5a39ee7bf008864556f27870615b9., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=75a5a39ee7bf008864556f27870615b9, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:00:30,855 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34392 deadline: 1731751290853, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=75a5a39ee7bf008864556f27870615b9, server=037c58a19ddb,37267,1731751077844 2024-11-16T10:00:30,855 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731751217321.75a5a39ee7bf008864556f27870615b9., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 because the exception is null or not the one we care about 2024-11-16T10:00:30,856 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=75a5a39ee7bf008864556f27870615b9, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T10:00:30,856 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731751217321.75a5a39ee7bf008864556f27870615b9., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 , the old value is region=TestAcidGuarantees,,1731751217321.75a5a39ee7bf008864556f27870615b9., hostname=037c58a19ddb,37267,1731751077844, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=75a5a39ee7bf008864556f27870615b9, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:00:30,856 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731751217321.75a5a39ee7bf008864556f27870615b9., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=75a5a39ee7bf008864556f27870615b9, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:00:30,856 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34430 deadline: 1731751290853, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=75a5a39ee7bf008864556f27870615b9, server=037c58a19ddb,37267,1731751077844 2024-11-16T10:00:30,856 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731751217321.75a5a39ee7bf008864556f27870615b9., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 because the exception is null or not the one we care about 2024-11-16T10:00:30,856 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=75a5a39ee7bf008864556f27870615b9, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T10:00:30,856 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34428 deadline: 1731751290853, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=75a5a39ee7bf008864556f27870615b9, server=037c58a19ddb,37267,1731751077844 2024-11-16T10:00:30,856 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731751217321.75a5a39ee7bf008864556f27870615b9., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 , the old value is region=TestAcidGuarantees,,1731751217321.75a5a39ee7bf008864556f27870615b9., hostname=037c58a19ddb,37267,1731751077844, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=75a5a39ee7bf008864556f27870615b9, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:00:30,856 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731751217321.75a5a39ee7bf008864556f27870615b9., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=75a5a39ee7bf008864556f27870615b9, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:00:30,856 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731751217321.75a5a39ee7bf008864556f27870615b9., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 because the exception is null or not the one we care about 2024-11-16T10:00:30,856 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=75a5a39ee7bf008864556f27870615b9, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T10:00:30,856 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34454 deadline: 1731751290853, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=75a5a39ee7bf008864556f27870615b9, server=037c58a19ddb,37267,1731751077844 2024-11-16T10:00:30,856 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731751217321.75a5a39ee7bf008864556f27870615b9., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 , the old value is region=TestAcidGuarantees,,1731751217321.75a5a39ee7bf008864556f27870615b9., hostname=037c58a19ddb,37267,1731751077844, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=75a5a39ee7bf008864556f27870615b9, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:00:30,856 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731751217321.75a5a39ee7bf008864556f27870615b9., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=75a5a39ee7bf008864556f27870615b9, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:00:30,856 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731751217321.75a5a39ee7bf008864556f27870615b9., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 because the exception is null or not the one we care about 2024-11-16T10:00:30,856 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731751217321.75a5a39ee7bf008864556f27870615b9., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 , the old value is region=TestAcidGuarantees,,1731751217321.75a5a39ee7bf008864556f27870615b9., hostname=037c58a19ddb,37267,1731751077844, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=75a5a39ee7bf008864556f27870615b9, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:00:30,856 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731751217321.75a5a39ee7bf008864556f27870615b9., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=75a5a39ee7bf008864556f27870615b9, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:00:30,856 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731751217321.75a5a39ee7bf008864556f27870615b9., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 because the exception is null or not the one we care about 2024-11-16T10:00:30,939 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=53.67 KB at sequenceid=197 (bloomFilter=true), to=hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/75a5a39ee7bf008864556f27870615b9/.tmp/C/9c5660694a7a480caf5c5fc83c189dc4 2024-11-16T10:00:30,943 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/75a5a39ee7bf008864556f27870615b9/.tmp/A/955ac682fed54cc0af1fa49e6c502dd2 as hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/75a5a39ee7bf008864556f27870615b9/A/955ac682fed54cc0af1fa49e6c502dd2 2024-11-16T10:00:30,946 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/75a5a39ee7bf008864556f27870615b9/A/955ac682fed54cc0af1fa49e6c502dd2, entries=200, sequenceid=197, filesize=38.8 K 2024-11-16T10:00:30,947 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/75a5a39ee7bf008864556f27870615b9/.tmp/B/eb93c6fdfe03475481ef8e2ff93de934 as hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/75a5a39ee7bf008864556f27870615b9/B/eb93c6fdfe03475481ef8e2ff93de934 2024-11-16T10:00:30,950 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/75a5a39ee7bf008864556f27870615b9/B/eb93c6fdfe03475481ef8e2ff93de934, entries=150, sequenceid=197, filesize=11.9 K 2024-11-16T10:00:30,950 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/75a5a39ee7bf008864556f27870615b9/.tmp/C/9c5660694a7a480caf5c5fc83c189dc4 as hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/75a5a39ee7bf008864556f27870615b9/C/9c5660694a7a480caf5c5fc83c189dc4 2024-11-16T10:00:30,953 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/75a5a39ee7bf008864556f27870615b9/C/9c5660694a7a480caf5c5fc83c189dc4, entries=150, sequenceid=197, filesize=11.9 K 2024-11-16T10:00:30,954 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~161.02 KB/164880, heapSize ~422.58 KB/432720, currentSize=46.96 KB/48090 for 75a5a39ee7bf008864556f27870615b9 in 1258ms, sequenceid=197, compaction requested=true 2024-11-16T10:00:30,954 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 75a5a39ee7bf008864556f27870615b9: 2024-11-16T10:00:30,954 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 75a5a39ee7bf008864556f27870615b9:A, priority=-2147483648, current under compaction store size is 1 2024-11-16T10:00:30,954 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-16T10:00:30,954 DEBUG [RS:0;037c58a19ddb:37267-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-16T10:00:30,954 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 75a5a39ee7bf008864556f27870615b9:B, priority=-2147483648, current under compaction store size is 2 2024-11-16T10:00:30,954 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-16T10:00:30,954 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 75a5a39ee7bf008864556f27870615b9:C, priority=-2147483648, current under compaction store size is 3 2024-11-16T10:00:30,954 DEBUG [RS:0;037c58a19ddb:37267-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-16T10:00:30,954 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-16T10:00:30,955 DEBUG [RS:0;037c58a19ddb:37267-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 141966 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-16T10:00:30,955 DEBUG [RS:0;037c58a19ddb:37267-shortCompactions-0 {}] regionserver.HStore(1541): 75a5a39ee7bf008864556f27870615b9/A is initiating minor compaction (all files) 2024-11-16T10:00:30,955 INFO [RS:0;037c58a19ddb:37267-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 75a5a39ee7bf008864556f27870615b9/A in TestAcidGuarantees,,1731751217321.75a5a39ee7bf008864556f27870615b9. 2024-11-16T10:00:30,955 INFO [RS:0;037c58a19ddb:37267-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/75a5a39ee7bf008864556f27870615b9/A/2567f08bfa9a407898aa25582d9a0dff, hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/75a5a39ee7bf008864556f27870615b9/A/80e1e8f18c254519b6be286bcb29f3db, hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/75a5a39ee7bf008864556f27870615b9/A/5ba81c6f8c6544a29da2e79b2c8b49d0, hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/75a5a39ee7bf008864556f27870615b9/A/955ac682fed54cc0af1fa49e6c502dd2] into tmpdir=hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/75a5a39ee7bf008864556f27870615b9/.tmp, totalSize=138.6 K 2024-11-16T10:00:30,955 INFO [RS:0;037c58a19ddb:37267-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=12 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731751217321.75a5a39ee7bf008864556f27870615b9. 2024-11-16T10:00:30,955 DEBUG [RS:0;037c58a19ddb:37267-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731751217321.75a5a39ee7bf008864556f27870615b9. files: [hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/75a5a39ee7bf008864556f27870615b9/A/2567f08bfa9a407898aa25582d9a0dff, hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/75a5a39ee7bf008864556f27870615b9/A/80e1e8f18c254519b6be286bcb29f3db, hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/75a5a39ee7bf008864556f27870615b9/A/5ba81c6f8c6544a29da2e79b2c8b49d0, hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/75a5a39ee7bf008864556f27870615b9/A/955ac682fed54cc0af1fa49e6c502dd2] 2024-11-16T10:00:30,956 DEBUG [RS:0;037c58a19ddb:37267-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48862 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-16T10:00:30,956 DEBUG [RS:0;037c58a19ddb:37267-shortCompactions-0 {}] compactions.Compactor(225): Compacting 2567f08bfa9a407898aa25582d9a0dff, keycount=150, bloomtype=ROW, size=30.6 K, encoding=NONE, compression=NONE, seqNum=131, earliestPutTs=1731751227211 2024-11-16T10:00:30,956 DEBUG [RS:0;037c58a19ddb:37267-longCompactions-0 {}] regionserver.HStore(1541): 75a5a39ee7bf008864556f27870615b9/B is initiating minor compaction (all files) 2024-11-16T10:00:30,956 INFO [RS:0;037c58a19ddb:37267-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 75a5a39ee7bf008864556f27870615b9/B in TestAcidGuarantees,,1731751217321.75a5a39ee7bf008864556f27870615b9. 2024-11-16T10:00:30,956 INFO [RS:0;037c58a19ddb:37267-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/75a5a39ee7bf008864556f27870615b9/B/53b261f643444e2f9eeda0200e77f83c, hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/75a5a39ee7bf008864556f27870615b9/B/36b2600aa11f4987924ad638f9bf96a5, hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/75a5a39ee7bf008864556f27870615b9/B/0cef038cd17c43eab448fd42a00414aa, hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/75a5a39ee7bf008864556f27870615b9/B/eb93c6fdfe03475481ef8e2ff93de934] into tmpdir=hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/75a5a39ee7bf008864556f27870615b9/.tmp, totalSize=47.7 K 2024-11-16T10:00:30,956 DEBUG [RS:0;037c58a19ddb:37267-shortCompactions-0 {}] compactions.Compactor(225): Compacting 80e1e8f18c254519b6be286bcb29f3db, keycount=200, bloomtype=ROW, size=38.8 K, encoding=NONE, compression=NONE, seqNum=156, earliestPutTs=1731751227246 2024-11-16T10:00:30,956 DEBUG [RS:0;037c58a19ddb:37267-longCompactions-0 {}] compactions.Compactor(225): Compacting 53b261f643444e2f9eeda0200e77f83c, keycount=150, bloomtype=ROW, size=12.1 K, encoding=NONE, compression=NONE, seqNum=131, earliestPutTs=1731751227211 2024-11-16T10:00:30,957 DEBUG [RS:0;037c58a19ddb:37267-shortCompactions-0 {}] compactions.Compactor(225): Compacting 5ba81c6f8c6544a29da2e79b2c8b49d0, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=170, earliestPutTs=1731751227379 2024-11-16T10:00:30,957 DEBUG [RS:0;037c58a19ddb:37267-longCompactions-0 {}] compactions.Compactor(225): Compacting 36b2600aa11f4987924ad638f9bf96a5, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=156, earliestPutTs=1731751227246 2024-11-16T10:00:30,957 DEBUG [RS:0;037c58a19ddb:37267-longCompactions-0 {}] compactions.Compactor(225): Compacting 0cef038cd17c43eab448fd42a00414aa, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=170, earliestPutTs=1731751227379 2024-11-16T10:00:30,957 DEBUG [RS:0;037c58a19ddb:37267-shortCompactions-0 {}] compactions.Compactor(225): Compacting 955ac682fed54cc0af1fa49e6c502dd2, keycount=200, bloomtype=ROW, size=38.8 K, encoding=NONE, compression=NONE, seqNum=197, earliestPutTs=1731751229586 2024-11-16T10:00:30,957 DEBUG [RS:0;037c58a19ddb:37267-longCompactions-0 {}] compactions.Compactor(225): Compacting eb93c6fdfe03475481ef8e2ff93de934, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=197, earliestPutTs=1731751229586 2024-11-16T10:00:30,964 INFO [RS:0;037c58a19ddb:37267-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=75a5a39ee7bf008864556f27870615b9] 2024-11-16T10:00:30,964 INFO [RS:0;037c58a19ddb:37267-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 75a5a39ee7bf008864556f27870615b9#B#compaction#505 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-16T10:00:30,965 DEBUG [RS:0;037c58a19ddb:37267-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/75a5a39ee7bf008864556f27870615b9/.tmp/B/d4e00a75365b4c6e937dcf823b92da7c is 50, key is test_row_0/B:col10/1731751229586/Put/seqid=0 2024-11-16T10:00:30,968 DEBUG [RS:0;037c58a19ddb:37267-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241116362543bd889b4949be156bb103bac883_75a5a39ee7bf008864556f27870615b9 store=[table=TestAcidGuarantees family=A region=75a5a39ee7bf008864556f27870615b9] 2024-11-16T10:00:30,970 DEBUG [RS:0;037c58a19ddb:37267-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241116362543bd889b4949be156bb103bac883_75a5a39ee7bf008864556f27870615b9, store=[table=TestAcidGuarantees family=A region=75a5a39ee7bf008864556f27870615b9] 2024-11-16T10:00:30,970 DEBUG [RS:0;037c58a19ddb:37267-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241116362543bd889b4949be156bb103bac883_75a5a39ee7bf008864556f27870615b9 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=75a5a39ee7bf008864556f27870615b9] 2024-11-16T10:00:30,977 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37591 is added to blk_1073742419_1595 (size=12595) 2024-11-16T10:00:30,981 DEBUG [RS:0;037c58a19ddb:37267-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/75a5a39ee7bf008864556f27870615b9/.tmp/B/d4e00a75365b4c6e937dcf823b92da7c as hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/75a5a39ee7bf008864556f27870615b9/B/d4e00a75365b4c6e937dcf823b92da7c 2024-11-16T10:00:30,984 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37591 is added to blk_1073742420_1596 (size=4469) 2024-11-16T10:00:30,985 INFO [RS:0;037c58a19ddb:37267-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 75a5a39ee7bf008864556f27870615b9#A#compaction#506 average throughput is 1.16 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-16T10:00:30,986 DEBUG [RS:0;037c58a19ddb:37267-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/75a5a39ee7bf008864556f27870615b9/.tmp/A/b9bdffdf7c5d49568463afe11efd7753 is 175, key is test_row_0/A:col10/1731751229586/Put/seqid=0 2024-11-16T10:00:30,988 INFO [RS:0;037c58a19ddb:37267-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 75a5a39ee7bf008864556f27870615b9/B of 75a5a39ee7bf008864556f27870615b9 into d4e00a75365b4c6e937dcf823b92da7c(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-16T10:00:30,988 DEBUG [RS:0;037c58a19ddb:37267-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 75a5a39ee7bf008864556f27870615b9: 2024-11-16T10:00:30,988 INFO [RS:0;037c58a19ddb:37267-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731751217321.75a5a39ee7bf008864556f27870615b9., storeName=75a5a39ee7bf008864556f27870615b9/B, priority=12, startTime=1731751230954; duration=0sec 2024-11-16T10:00:30,988 DEBUG [RS:0;037c58a19ddb:37267-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-16T10:00:30,988 DEBUG [RS:0;037c58a19ddb:37267-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 75a5a39ee7bf008864556f27870615b9:B 2024-11-16T10:00:30,988 DEBUG [RS:0;037c58a19ddb:37267-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-16T10:00:30,989 DEBUG [RS:0;037c58a19ddb:37267-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48862 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-16T10:00:30,989 DEBUG [RS:0;037c58a19ddb:37267-longCompactions-0 {}] regionserver.HStore(1541): 75a5a39ee7bf008864556f27870615b9/C is initiating minor compaction (all files) 2024-11-16T10:00:30,989 INFO [RS:0;037c58a19ddb:37267-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 75a5a39ee7bf008864556f27870615b9/C in TestAcidGuarantees,,1731751217321.75a5a39ee7bf008864556f27870615b9. 2024-11-16T10:00:30,989 INFO [RS:0;037c58a19ddb:37267-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/75a5a39ee7bf008864556f27870615b9/C/22cf205d63134391b67b77b703ef814d, hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/75a5a39ee7bf008864556f27870615b9/C/38bbc22a1129485f90b4650be6aa0326, hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/75a5a39ee7bf008864556f27870615b9/C/eece82162ad545eea9eedfee8e3109fc, hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/75a5a39ee7bf008864556f27870615b9/C/9c5660694a7a480caf5c5fc83c189dc4] into tmpdir=hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/75a5a39ee7bf008864556f27870615b9/.tmp, totalSize=47.7 K 2024-11-16T10:00:30,990 DEBUG [RS:0;037c58a19ddb:37267-longCompactions-0 {}] compactions.Compactor(225): Compacting 22cf205d63134391b67b77b703ef814d, keycount=150, bloomtype=ROW, size=12.1 K, encoding=NONE, compression=NONE, seqNum=131, earliestPutTs=1731751227211 2024-11-16T10:00:30,990 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37591 is added to blk_1073742421_1597 (size=31549) 2024-11-16T10:00:30,990 DEBUG [RS:0;037c58a19ddb:37267-longCompactions-0 {}] compactions.Compactor(225): Compacting 38bbc22a1129485f90b4650be6aa0326, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=156, earliestPutTs=1731751227246 2024-11-16T10:00:30,991 DEBUG [RS:0;037c58a19ddb:37267-longCompactions-0 {}] compactions.Compactor(225): Compacting eece82162ad545eea9eedfee8e3109fc, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=170, earliestPutTs=1731751227379 2024-11-16T10:00:30,991 DEBUG [RS:0;037c58a19ddb:37267-longCompactions-0 {}] compactions.Compactor(225): Compacting 9c5660694a7a480caf5c5fc83c189dc4, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=197, earliestPutTs=1731751229586 2024-11-16T10:00:30,995 DEBUG [RS:0;037c58a19ddb:37267-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/75a5a39ee7bf008864556f27870615b9/.tmp/A/b9bdffdf7c5d49568463afe11efd7753 as hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/75a5a39ee7bf008864556f27870615b9/A/b9bdffdf7c5d49568463afe11efd7753 2024-11-16T10:00:30,999 INFO [RS:0;037c58a19ddb:37267-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 75a5a39ee7bf008864556f27870615b9/A of 75a5a39ee7bf008864556f27870615b9 into b9bdffdf7c5d49568463afe11efd7753(size=30.8 K), total size for store is 30.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-16T10:00:30,999 INFO [RS:0;037c58a19ddb:37267-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 75a5a39ee7bf008864556f27870615b9#C#compaction#507 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-16T10:00:30,999 DEBUG [RS:0;037c58a19ddb:37267-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 75a5a39ee7bf008864556f27870615b9: 2024-11-16T10:00:30,999 INFO [RS:0;037c58a19ddb:37267-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731751217321.75a5a39ee7bf008864556f27870615b9., storeName=75a5a39ee7bf008864556f27870615b9/A, priority=12, startTime=1731751230954; duration=0sec 2024-11-16T10:00:31,000 DEBUG [RS:0;037c58a19ddb:37267-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-16T10:00:31,000 DEBUG [RS:0;037c58a19ddb:37267-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 75a5a39ee7bf008864556f27870615b9:A 2024-11-16T10:00:31,000 DEBUG [RS:0;037c58a19ddb:37267-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/75a5a39ee7bf008864556f27870615b9/.tmp/C/bb43f99d4a98465cbfa41dff2bac86f9 is 50, key is test_row_0/C:col10/1731751229586/Put/seqid=0 2024-11-16T10:00:31,003 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37591 is added to blk_1073742422_1598 (size=12595) 2024-11-16T10:00:31,233 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37877 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=156 2024-11-16T10:00:31,233 INFO [RPCClient-NioEventLoopGroup-4-12 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-16T10:00:31,234 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37877 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-16T10:00:31,235 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37877 {}] procedure2.ProcedureExecutor(1139): Stored pid=158, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=158, table=TestAcidGuarantees 2024-11-16T10:00:31,235 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37877 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=158 2024-11-16T10:00:31,236 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=158, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=158, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-16T10:00:31,236 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=158, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=158, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-16T10:00:31,236 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=159, ppid=158, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-16T10:00:31,343 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37877 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=158 2024-11-16T10:00:31,388 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=37267 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=159 2024-11-16T10:00:31,389 DEBUG [RS_FLUSH_OPERATIONS-regionserver/037c58a19ddb:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731751217321.75a5a39ee7bf008864556f27870615b9. 2024-11-16T10:00:31,389 INFO [RS_FLUSH_OPERATIONS-regionserver/037c58a19ddb:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HRegion(2902): Flushing 75a5a39ee7bf008864556f27870615b9 3/3 column families, dataSize=46.96 KB heapSize=123.80 KB 2024-11-16T10:00:31,395 DEBUG [RS_FLUSH_OPERATIONS-regionserver/037c58a19ddb:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241116cb1ff8adb3814d90a1664ea7e7d28601_75a5a39ee7bf008864556f27870615b9 is 50, key is test_row_0/A:col10/1731751229705/Put/seqid=0 2024-11-16T10:00:31,398 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37591 is added to blk_1073742423_1599 (size=12304) 2024-11-16T10:00:31,407 DEBUG [RS:0;037c58a19ddb:37267-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/75a5a39ee7bf008864556f27870615b9/.tmp/C/bb43f99d4a98465cbfa41dff2bac86f9 as hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/75a5a39ee7bf008864556f27870615b9/C/bb43f99d4a98465cbfa41dff2bac86f9 2024-11-16T10:00:31,411 INFO [RS:0;037c58a19ddb:37267-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 75a5a39ee7bf008864556f27870615b9/C of 75a5a39ee7bf008864556f27870615b9 into bb43f99d4a98465cbfa41dff2bac86f9(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-16T10:00:31,411 DEBUG [RS:0;037c58a19ddb:37267-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 75a5a39ee7bf008864556f27870615b9: 2024-11-16T10:00:31,411 INFO [RS:0;037c58a19ddb:37267-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731751217321.75a5a39ee7bf008864556f27870615b9., storeName=75a5a39ee7bf008864556f27870615b9/C, priority=12, startTime=1731751230954; duration=0sec 2024-11-16T10:00:31,411 DEBUG [RS:0;037c58a19ddb:37267-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-16T10:00:31,411 DEBUG [RS:0;037c58a19ddb:37267-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 75a5a39ee7bf008864556f27870615b9:C 2024-11-16T10:00:31,553 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37877 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=158 2024-11-16T10:00:31,799 DEBUG [RS_FLUSH_OPERATIONS-regionserver/037c58a19ddb:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:31,803 INFO [RS_FLUSH_OPERATIONS-regionserver/037c58a19ddb:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241116cb1ff8adb3814d90a1664ea7e7d28601_75a5a39ee7bf008864556f27870615b9 to hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241116cb1ff8adb3814d90a1664ea7e7d28601_75a5a39ee7bf008864556f27870615b9 2024-11-16T10:00:31,804 DEBUG [RS_FLUSH_OPERATIONS-regionserver/037c58a19ddb:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/75a5a39ee7bf008864556f27870615b9/.tmp/A/584e3cb61e0b4e15acdabe73ca47b987, store: [table=TestAcidGuarantees family=A region=75a5a39ee7bf008864556f27870615b9] 2024-11-16T10:00:31,804 DEBUG [RS_FLUSH_OPERATIONS-regionserver/037c58a19ddb:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/75a5a39ee7bf008864556f27870615b9/.tmp/A/584e3cb61e0b4e15acdabe73ca47b987 is 175, key is test_row_0/A:col10/1731751229705/Put/seqid=0 2024-11-16T10:00:31,807 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37591 is added to blk_1073742424_1600 (size=31105) 2024-11-16T10:00:31,808 INFO [RS_FLUSH_OPERATIONS-regionserver/037c58a19ddb:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=209, memsize=15.7 K, hasBloomFilter=true, into tmp file hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/75a5a39ee7bf008864556f27870615b9/.tmp/A/584e3cb61e0b4e15acdabe73ca47b987 2024-11-16T10:00:31,814 DEBUG [RS_FLUSH_OPERATIONS-regionserver/037c58a19ddb:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/75a5a39ee7bf008864556f27870615b9/.tmp/B/fdace4a2305248da8f37008895cb2774 is 50, key is test_row_0/B:col10/1731751229705/Put/seqid=0 2024-11-16T10:00:31,817 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37591 is added to blk_1073742425_1601 (size=12151) 2024-11-16T10:00:31,863 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37877 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=158 2024-11-16T10:00:31,864 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] regionserver.HRegion(8855): Flush requested on 75a5a39ee7bf008864556f27870615b9 2024-11-16T10:00:31,865 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731751217321.75a5a39ee7bf008864556f27870615b9. as already flushing 2024-11-16T10:00:31,894 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=75a5a39ee7bf008864556f27870615b9, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T10:00:31,894 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34454 deadline: 1731751291888, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=75a5a39ee7bf008864556f27870615b9, server=037c58a19ddb,37267,1731751077844 2024-11-16T10:00:31,894 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=75a5a39ee7bf008864556f27870615b9, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T10:00:31,894 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731751217321.75a5a39ee7bf008864556f27870615b9., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 , the old value is region=TestAcidGuarantees,,1731751217321.75a5a39ee7bf008864556f27870615b9., hostname=037c58a19ddb,37267,1731751077844, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=75a5a39ee7bf008864556f27870615b9, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:00:31,894 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731751217321.75a5a39ee7bf008864556f27870615b9., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=75a5a39ee7bf008864556f27870615b9, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:00:31,894 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] ipc.CallRunner(138): callId: 73 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34412 deadline: 1731751291889, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=75a5a39ee7bf008864556f27870615b9, server=037c58a19ddb,37267,1731751077844 2024-11-16T10:00:31,894 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731751217321.75a5a39ee7bf008864556f27870615b9., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 because the exception is null or not the one we care about 2024-11-16T10:00:31,894 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=75a5a39ee7bf008864556f27870615b9, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T10:00:31,895 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34392 deadline: 1731751291890, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=75a5a39ee7bf008864556f27870615b9, server=037c58a19ddb,37267,1731751077844 2024-11-16T10:00:31,895 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731751217321.75a5a39ee7bf008864556f27870615b9., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 , the old value is region=TestAcidGuarantees,,1731751217321.75a5a39ee7bf008864556f27870615b9., hostname=037c58a19ddb,37267,1731751077844, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=75a5a39ee7bf008864556f27870615b9, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:00:31,895 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731751217321.75a5a39ee7bf008864556f27870615b9., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=75a5a39ee7bf008864556f27870615b9, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:00:31,895 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731751217321.75a5a39ee7bf008864556f27870615b9., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 because the exception is null or not the one we care about 2024-11-16T10:00:31,895 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731751217321.75a5a39ee7bf008864556f27870615b9., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 , the old value is region=TestAcidGuarantees,,1731751217321.75a5a39ee7bf008864556f27870615b9., hostname=037c58a19ddb,37267,1731751077844, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=75a5a39ee7bf008864556f27870615b9, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:00:31,895 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=75a5a39ee7bf008864556f27870615b9, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T10:00:31,895 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731751217321.75a5a39ee7bf008864556f27870615b9., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=75a5a39ee7bf008864556f27870615b9, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:00:31,895 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731751217321.75a5a39ee7bf008864556f27870615b9., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 because the exception is null or not the one we care about 2024-11-16T10:00:31,895 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34428 deadline: 1731751291891, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=75a5a39ee7bf008864556f27870615b9, server=037c58a19ddb,37267,1731751077844 2024-11-16T10:00:31,895 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=75a5a39ee7bf008864556f27870615b9, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T10:00:31,895 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34430 deadline: 1731751291891, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=75a5a39ee7bf008864556f27870615b9, server=037c58a19ddb,37267,1731751077844 2024-11-16T10:00:31,895 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731751217321.75a5a39ee7bf008864556f27870615b9., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 , the old value is region=TestAcidGuarantees,,1731751217321.75a5a39ee7bf008864556f27870615b9., hostname=037c58a19ddb,37267,1731751077844, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=75a5a39ee7bf008864556f27870615b9, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:00:31,895 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731751217321.75a5a39ee7bf008864556f27870615b9., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=75a5a39ee7bf008864556f27870615b9, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:00:31,895 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731751217321.75a5a39ee7bf008864556f27870615b9., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 because the exception is null or not the one we care about 2024-11-16T10:00:31,895 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731751217321.75a5a39ee7bf008864556f27870615b9., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 , the old value is region=TestAcidGuarantees,,1731751217321.75a5a39ee7bf008864556f27870615b9., hostname=037c58a19ddb,37267,1731751077844, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=75a5a39ee7bf008864556f27870615b9, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:00:31,895 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731751217321.75a5a39ee7bf008864556f27870615b9., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=75a5a39ee7bf008864556f27870615b9, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:00:31,895 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731751217321.75a5a39ee7bf008864556f27870615b9., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 because the exception is null or not the one we care about 2024-11-16T10:00:32,005 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=75a5a39ee7bf008864556f27870615b9, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T10:00:32,005 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34454 deadline: 1731751292003, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=75a5a39ee7bf008864556f27870615b9, server=037c58a19ddb,37267,1731751077844 2024-11-16T10:00:32,006 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=75a5a39ee7bf008864556f27870615b9, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T10:00:32,006 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731751217321.75a5a39ee7bf008864556f27870615b9., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 , the old value is region=TestAcidGuarantees,,1731751217321.75a5a39ee7bf008864556f27870615b9., hostname=037c58a19ddb,37267,1731751077844, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=75a5a39ee7bf008864556f27870615b9, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:00:32,006 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] ipc.CallRunner(138): callId: 74 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34412 deadline: 1731751292003, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=75a5a39ee7bf008864556f27870615b9, server=037c58a19ddb,37267,1731751077844 2024-11-16T10:00:32,006 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731751217321.75a5a39ee7bf008864556f27870615b9., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=75a5a39ee7bf008864556f27870615b9, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:00:32,006 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731751217321.75a5a39ee7bf008864556f27870615b9., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 because the exception is null or not the one we care about 2024-11-16T10:00:32,006 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=75a5a39ee7bf008864556f27870615b9, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T10:00:32,006 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731751217321.75a5a39ee7bf008864556f27870615b9., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 , the old value is region=TestAcidGuarantees,,1731751217321.75a5a39ee7bf008864556f27870615b9., hostname=037c58a19ddb,37267,1731751077844, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=75a5a39ee7bf008864556f27870615b9, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:00:32,006 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731751217321.75a5a39ee7bf008864556f27870615b9., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=75a5a39ee7bf008864556f27870615b9, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:00:32,006 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34392 deadline: 1731751292003, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=75a5a39ee7bf008864556f27870615b9, server=037c58a19ddb,37267,1731751077844 2024-11-16T10:00:32,006 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731751217321.75a5a39ee7bf008864556f27870615b9., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 because the exception is null or not the one we care about 2024-11-16T10:00:32,006 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=75a5a39ee7bf008864556f27870615b9, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T10:00:32,006 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=75a5a39ee7bf008864556f27870615b9, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T10:00:32,006 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731751217321.75a5a39ee7bf008864556f27870615b9., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 , the old value is region=TestAcidGuarantees,,1731751217321.75a5a39ee7bf008864556f27870615b9., hostname=037c58a19ddb,37267,1731751077844, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=75a5a39ee7bf008864556f27870615b9, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:00:32,006 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34428 deadline: 1731751292003, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=75a5a39ee7bf008864556f27870615b9, server=037c58a19ddb,37267,1731751077844 2024-11-16T10:00:32,006 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731751217321.75a5a39ee7bf008864556f27870615b9., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=75a5a39ee7bf008864556f27870615b9, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:00:32,006 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731751217321.75a5a39ee7bf008864556f27870615b9., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 because the exception is null or not the one we care about 2024-11-16T10:00:32,006 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] ipc.CallRunner(138): callId: 71 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34430 deadline: 1731751292003, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=75a5a39ee7bf008864556f27870615b9, server=037c58a19ddb,37267,1731751077844 2024-11-16T10:00:32,007 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731751217321.75a5a39ee7bf008864556f27870615b9., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 , the old value is region=TestAcidGuarantees,,1731751217321.75a5a39ee7bf008864556f27870615b9., hostname=037c58a19ddb,37267,1731751077844, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=75a5a39ee7bf008864556f27870615b9, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:00:32,007 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731751217321.75a5a39ee7bf008864556f27870615b9., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=75a5a39ee7bf008864556f27870615b9, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:00:32,007 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731751217321.75a5a39ee7bf008864556f27870615b9., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 because the exception is null or not the one we care about 2024-11-16T10:00:32,007 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731751217321.75a5a39ee7bf008864556f27870615b9., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 , the old value is region=TestAcidGuarantees,,1731751217321.75a5a39ee7bf008864556f27870615b9., hostname=037c58a19ddb,37267,1731751077844, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=75a5a39ee7bf008864556f27870615b9, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:00:32,007 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731751217321.75a5a39ee7bf008864556f27870615b9., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=75a5a39ee7bf008864556f27870615b9, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:00:32,007 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731751217321.75a5a39ee7bf008864556f27870615b9., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 because the exception is null or not the one we care about 2024-11-16T10:00:32,213 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=75a5a39ee7bf008864556f27870615b9, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T10:00:32,213 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34454 deadline: 1731751292212, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=75a5a39ee7bf008864556f27870615b9, server=037c58a19ddb,37267,1731751077844 2024-11-16T10:00:32,213 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=75a5a39ee7bf008864556f27870615b9, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T10:00:32,213 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731751217321.75a5a39ee7bf008864556f27870615b9., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 , the old value is region=TestAcidGuarantees,,1731751217321.75a5a39ee7bf008864556f27870615b9., hostname=037c58a19ddb,37267,1731751077844, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=75a5a39ee7bf008864556f27870615b9, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:00:32,214 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731751217321.75a5a39ee7bf008864556f27870615b9., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=75a5a39ee7bf008864556f27870615b9, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:00:32,214 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] ipc.CallRunner(138): callId: 75 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34412 deadline: 1731751292212, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=75a5a39ee7bf008864556f27870615b9, server=037c58a19ddb,37267,1731751077844 2024-11-16T10:00:32,214 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731751217321.75a5a39ee7bf008864556f27870615b9., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 because the exception is null or not the one we care about 2024-11-16T10:00:32,214 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731751217321.75a5a39ee7bf008864556f27870615b9., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 , the old value is region=TestAcidGuarantees,,1731751217321.75a5a39ee7bf008864556f27870615b9., hostname=037c58a19ddb,37267,1731751077844, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=75a5a39ee7bf008864556f27870615b9, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:00:32,214 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731751217321.75a5a39ee7bf008864556f27870615b9., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=75a5a39ee7bf008864556f27870615b9, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:00:32,214 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731751217321.75a5a39ee7bf008864556f27870615b9., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 because the exception is null or not the one we care about 2024-11-16T10:00:32,214 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=75a5a39ee7bf008864556f27870615b9, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T10:00:32,214 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34392 deadline: 1731751292213, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=75a5a39ee7bf008864556f27870615b9, server=037c58a19ddb,37267,1731751077844 2024-11-16T10:00:32,214 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731751217321.75a5a39ee7bf008864556f27870615b9., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 , the old value is region=TestAcidGuarantees,,1731751217321.75a5a39ee7bf008864556f27870615b9., hostname=037c58a19ddb,37267,1731751077844, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=75a5a39ee7bf008864556f27870615b9, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:00:32,214 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731751217321.75a5a39ee7bf008864556f27870615b9., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=75a5a39ee7bf008864556f27870615b9, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:00:32,214 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731751217321.75a5a39ee7bf008864556f27870615b9., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 because the exception is null or not the one we care about 2024-11-16T10:00:32,214 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=75a5a39ee7bf008864556f27870615b9, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T10:00:32,214 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34428 deadline: 1731751292213, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=75a5a39ee7bf008864556f27870615b9, server=037c58a19ddb,37267,1731751077844 2024-11-16T10:00:32,214 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731751217321.75a5a39ee7bf008864556f27870615b9., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 , the old value is region=TestAcidGuarantees,,1731751217321.75a5a39ee7bf008864556f27870615b9., hostname=037c58a19ddb,37267,1731751077844, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=75a5a39ee7bf008864556f27870615b9, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:00:32,214 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731751217321.75a5a39ee7bf008864556f27870615b9., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=75a5a39ee7bf008864556f27870615b9, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:00:32,214 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731751217321.75a5a39ee7bf008864556f27870615b9., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 because the exception is null or not the one we care about 2024-11-16T10:00:32,214 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=75a5a39ee7bf008864556f27870615b9, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T10:00:32,215 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] ipc.CallRunner(138): callId: 72 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34430 deadline: 1731751292213, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=75a5a39ee7bf008864556f27870615b9, server=037c58a19ddb,37267,1731751077844 2024-11-16T10:00:32,215 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731751217321.75a5a39ee7bf008864556f27870615b9., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 , the old value is region=TestAcidGuarantees,,1731751217321.75a5a39ee7bf008864556f27870615b9., hostname=037c58a19ddb,37267,1731751077844, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=75a5a39ee7bf008864556f27870615b9, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:00:32,215 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731751217321.75a5a39ee7bf008864556f27870615b9., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=75a5a39ee7bf008864556f27870615b9, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:00:32,215 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731751217321.75a5a39ee7bf008864556f27870615b9., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 because the exception is null or not the one we care about 2024-11-16T10:00:32,218 INFO [RS_FLUSH_OPERATIONS-regionserver/037c58a19ddb:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=209 (bloomFilter=true), to=hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/75a5a39ee7bf008864556f27870615b9/.tmp/B/fdace4a2305248da8f37008895cb2774 2024-11-16T10:00:32,224 DEBUG [RS_FLUSH_OPERATIONS-regionserver/037c58a19ddb:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/75a5a39ee7bf008864556f27870615b9/.tmp/C/39ac7213ec73453dabe0a6d1fbad50a7 is 50, key is test_row_0/C:col10/1731751229705/Put/seqid=0 2024-11-16T10:00:32,234 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37591 is added to blk_1073742426_1602 (size=12151) 2024-11-16T10:00:32,235 INFO [RS_FLUSH_OPERATIONS-regionserver/037c58a19ddb:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=209 (bloomFilter=true), to=hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/75a5a39ee7bf008864556f27870615b9/.tmp/C/39ac7213ec73453dabe0a6d1fbad50a7 2024-11-16T10:00:32,239 DEBUG [RS_FLUSH_OPERATIONS-regionserver/037c58a19ddb:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/75a5a39ee7bf008864556f27870615b9/.tmp/A/584e3cb61e0b4e15acdabe73ca47b987 as hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/75a5a39ee7bf008864556f27870615b9/A/584e3cb61e0b4e15acdabe73ca47b987 2024-11-16T10:00:32,242 INFO [RS_FLUSH_OPERATIONS-regionserver/037c58a19ddb:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/75a5a39ee7bf008864556f27870615b9/A/584e3cb61e0b4e15acdabe73ca47b987, entries=150, sequenceid=209, filesize=30.4 K 2024-11-16T10:00:32,243 DEBUG [RS_FLUSH_OPERATIONS-regionserver/037c58a19ddb:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/75a5a39ee7bf008864556f27870615b9/.tmp/B/fdace4a2305248da8f37008895cb2774 as hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/75a5a39ee7bf008864556f27870615b9/B/fdace4a2305248da8f37008895cb2774 2024-11-16T10:00:32,246 INFO [RS_FLUSH_OPERATIONS-regionserver/037c58a19ddb:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/75a5a39ee7bf008864556f27870615b9/B/fdace4a2305248da8f37008895cb2774, entries=150, sequenceid=209, filesize=11.9 K 2024-11-16T10:00:32,246 DEBUG [RS_FLUSH_OPERATIONS-regionserver/037c58a19ddb:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/75a5a39ee7bf008864556f27870615b9/.tmp/C/39ac7213ec73453dabe0a6d1fbad50a7 as hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/75a5a39ee7bf008864556f27870615b9/C/39ac7213ec73453dabe0a6d1fbad50a7 2024-11-16T10:00:32,249 INFO [RS_FLUSH_OPERATIONS-regionserver/037c58a19ddb:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/75a5a39ee7bf008864556f27870615b9/C/39ac7213ec73453dabe0a6d1fbad50a7, entries=150, sequenceid=209, filesize=11.9 K 2024-11-16T10:00:32,250 INFO [RS_FLUSH_OPERATIONS-regionserver/037c58a19ddb:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HRegion(3140): Finished flush of dataSize ~46.96 KB/48090, heapSize ~123.75 KB/126720, currentSize=154.31 KB/158010 for 75a5a39ee7bf008864556f27870615b9 in 861ms, sequenceid=209, compaction requested=false 2024-11-16T10:00:32,250 DEBUG [RS_FLUSH_OPERATIONS-regionserver/037c58a19ddb:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HRegion(2603): Flush status journal for 75a5a39ee7bf008864556f27870615b9: 2024-11-16T10:00:32,250 DEBUG [RS_FLUSH_OPERATIONS-regionserver/037c58a19ddb:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731751217321.75a5a39ee7bf008864556f27870615b9. 2024-11-16T10:00:32,250 DEBUG [RS_FLUSH_OPERATIONS-regionserver/037c58a19ddb:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=159 2024-11-16T10:00:32,251 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37877 {}] master.HMaster(4169): Remote procedure done, pid=159 2024-11-16T10:00:32,254 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=159, resume processing ppid=158 2024-11-16T10:00:32,254 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=159, ppid=158, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.0160 sec 2024-11-16T10:00:32,256 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=158, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=158, table=TestAcidGuarantees in 1.0200 sec 2024-11-16T10:00:32,373 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37877 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=158 2024-11-16T10:00:32,373 INFO [RPCClient-NioEventLoopGroup-4-12 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-16T10:00:32,374 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37877 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-16T10:00:32,375 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37877 {}] procedure2.ProcedureExecutor(1139): Stored pid=160, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=160, table=TestAcidGuarantees 2024-11-16T10:00:32,376 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37877 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=160 2024-11-16T10:00:32,376 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=160, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=160, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-16T10:00:32,376 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=160, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=160, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-16T10:00:32,377 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=161, ppid=160, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-16T10:00:32,483 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37877 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=160 2024-11-16T10:00:32,524 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] regionserver.HRegion(8855): Flush requested on 75a5a39ee7bf008864556f27870615b9 2024-11-16T10:00:32,524 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-16T10:00:32,525 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 75a5a39ee7bf008864556f27870615b9 3/3 column families, dataSize=161.02 KB heapSize=422.63 KB 2024-11-16T10:00:32,529 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=37267 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=161 2024-11-16T10:00:32,529 DEBUG [RS_FLUSH_OPERATIONS-regionserver/037c58a19ddb:0-1 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731751217321.75a5a39ee7bf008864556f27870615b9. 2024-11-16T10:00:32,529 DEBUG [RS_FLUSH_OPERATIONS-regionserver/037c58a19ddb:0-1 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731751217321.75a5a39ee7bf008864556f27870615b9. as already flushing 2024-11-16T10:00:32,529 DEBUG [RS_FLUSH_OPERATIONS-regionserver/037c58a19ddb:0-1 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731751217321.75a5a39ee7bf008864556f27870615b9. 2024-11-16T10:00:32,529 ERROR [RS_FLUSH_OPERATIONS-regionserver/037c58a19ddb:0-1 {event_type=RS_FLUSH_REGIONS, pid=161}] handler.RSProcedureHandler(58): pid=161 java.io.IOException: Unable to complete flush {ENCODED => 75a5a39ee7bf008864556f27870615b9, NAME => 'TestAcidGuarantees,,1731751217321.75a5a39ee7bf008864556f27870615b9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T10:00:32,529 DEBUG [RS_FLUSH_OPERATIONS-regionserver/037c58a19ddb:0-1 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=161 java.io.IOException: Unable to complete flush {ENCODED => 75a5a39ee7bf008864556f27870615b9, NAME => 'TestAcidGuarantees,,1731751217321.75a5a39ee7bf008864556f27870615b9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T10:00:32,530 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37877 {}] master.HMaster(4177): Remote procedure failed, pid=161 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 75a5a39ee7bf008864556f27870615b9, NAME => 'TestAcidGuarantees,,1731751217321.75a5a39ee7bf008864556f27870615b9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 75a5a39ee7bf008864556f27870615b9, NAME => 'TestAcidGuarantees,,1731751217321.75a5a39ee7bf008864556f27870615b9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T10:00:32,532 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=75a5a39ee7bf008864556f27870615b9, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T10:00:32,532 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] ipc.CallRunner(138): callId: 72 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34392 deadline: 1731751292527, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=75a5a39ee7bf008864556f27870615b9, server=037c58a19ddb,37267,1731751077844 2024-11-16T10:00:32,532 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=75a5a39ee7bf008864556f27870615b9, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T10:00:32,532 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731751217321.75a5a39ee7bf008864556f27870615b9., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 , the old value is region=TestAcidGuarantees,,1731751217321.75a5a39ee7bf008864556f27870615b9., hostname=037c58a19ddb,37267,1731751077844, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=75a5a39ee7bf008864556f27870615b9, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:00:32,532 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731751217321.75a5a39ee7bf008864556f27870615b9., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=75a5a39ee7bf008864556f27870615b9, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:00:32,532 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34428 deadline: 1731751292527, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=75a5a39ee7bf008864556f27870615b9, server=037c58a19ddb,37267,1731751077844 2024-11-16T10:00:32,532 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731751217321.75a5a39ee7bf008864556f27870615b9., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 because the exception is null or not the one we care about 2024-11-16T10:00:32,532 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=75a5a39ee7bf008864556f27870615b9, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T10:00:32,532 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] ipc.CallRunner(138): callId: 74 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34430 deadline: 1731751292528, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=75a5a39ee7bf008864556f27870615b9, server=037c58a19ddb,37267,1731751077844 2024-11-16T10:00:32,533 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731751217321.75a5a39ee7bf008864556f27870615b9., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 , the old value is region=TestAcidGuarantees,,1731751217321.75a5a39ee7bf008864556f27870615b9., hostname=037c58a19ddb,37267,1731751077844, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=75a5a39ee7bf008864556f27870615b9, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:00:32,533 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731751217321.75a5a39ee7bf008864556f27870615b9., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=75a5a39ee7bf008864556f27870615b9, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:00:32,533 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=75a5a39ee7bf008864556f27870615b9, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T10:00:32,533 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731751217321.75a5a39ee7bf008864556f27870615b9., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 because the exception is null or not the one we care about 2024-11-16T10:00:32,533 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] ipc.CallRunner(138): callId: 71 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34454 deadline: 1731751292529, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=75a5a39ee7bf008864556f27870615b9, server=037c58a19ddb,37267,1731751077844 2024-11-16T10:00:32,533 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731751217321.75a5a39ee7bf008864556f27870615b9., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 , the old value is region=TestAcidGuarantees,,1731751217321.75a5a39ee7bf008864556f27870615b9., hostname=037c58a19ddb,37267,1731751077844, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=75a5a39ee7bf008864556f27870615b9, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:00:32,533 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731751217321.75a5a39ee7bf008864556f27870615b9., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=75a5a39ee7bf008864556f27870615b9, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:00:32,533 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731751217321.75a5a39ee7bf008864556f27870615b9., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 because the exception is null or not the one we care about 2024-11-16T10:00:32,533 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731751217321.75a5a39ee7bf008864556f27870615b9., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 , the old value is region=TestAcidGuarantees,,1731751217321.75a5a39ee7bf008864556f27870615b9., hostname=037c58a19ddb,37267,1731751077844, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=75a5a39ee7bf008864556f27870615b9, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:00:32,533 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731751217321.75a5a39ee7bf008864556f27870615b9., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=75a5a39ee7bf008864556f27870615b9, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:00:32,533 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731751217321.75a5a39ee7bf008864556f27870615b9., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 because the exception is null or not the one we care about 2024-11-16T10:00:32,533 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241116f3686b6e76ea42dbbf0fd2936f4e0b74_75a5a39ee7bf008864556f27870615b9 is 50, key is test_row_0/A:col10/1731751232523/Put/seqid=0 2024-11-16T10:00:32,535 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=75a5a39ee7bf008864556f27870615b9, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T10:00:32,535 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] ipc.CallRunner(138): callId: 78 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34412 deadline: 1731751292532, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=75a5a39ee7bf008864556f27870615b9, server=037c58a19ddb,37267,1731751077844 2024-11-16T10:00:32,536 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731751217321.75a5a39ee7bf008864556f27870615b9., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 , the old value is region=TestAcidGuarantees,,1731751217321.75a5a39ee7bf008864556f27870615b9., hostname=037c58a19ddb,37267,1731751077844, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=75a5a39ee7bf008864556f27870615b9, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:00:32,536 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731751217321.75a5a39ee7bf008864556f27870615b9., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=75a5a39ee7bf008864556f27870615b9, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:00:32,536 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731751217321.75a5a39ee7bf008864556f27870615b9., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 because the exception is null or not the one we care about 2024-11-16T10:00:32,538 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37591 is added to blk_1073742427_1603 (size=12304) 2024-11-16T10:00:32,646 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=75a5a39ee7bf008864556f27870615b9, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T10:00:32,646 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] ipc.CallRunner(138): callId: 73 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34392 deadline: 1731751292643, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=75a5a39ee7bf008864556f27870615b9, server=037c58a19ddb,37267,1731751077844 2024-11-16T10:00:32,646 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731751217321.75a5a39ee7bf008864556f27870615b9., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 , the old value is region=TestAcidGuarantees,,1731751217321.75a5a39ee7bf008864556f27870615b9., hostname=037c58a19ddb,37267,1731751077844, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=75a5a39ee7bf008864556f27870615b9, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:00:32,646 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731751217321.75a5a39ee7bf008864556f27870615b9., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=75a5a39ee7bf008864556f27870615b9, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:00:32,646 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=75a5a39ee7bf008864556f27870615b9, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T10:00:32,646 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731751217321.75a5a39ee7bf008864556f27870615b9., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 because the exception is null or not the one we care about 2024-11-16T10:00:32,646 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34428 deadline: 1731751292643, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=75a5a39ee7bf008864556f27870615b9, server=037c58a19ddb,37267,1731751077844 2024-11-16T10:00:32,646 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=75a5a39ee7bf008864556f27870615b9, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T10:00:32,646 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] ipc.CallRunner(138): callId: 75 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34430 deadline: 1731751292643, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=75a5a39ee7bf008864556f27870615b9, server=037c58a19ddb,37267,1731751077844 2024-11-16T10:00:32,646 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=75a5a39ee7bf008864556f27870615b9, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T10:00:32,646 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731751217321.75a5a39ee7bf008864556f27870615b9., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 , the old value is region=TestAcidGuarantees,,1731751217321.75a5a39ee7bf008864556f27870615b9., hostname=037c58a19ddb,37267,1731751077844, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=75a5a39ee7bf008864556f27870615b9, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:00:32,646 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731751217321.75a5a39ee7bf008864556f27870615b9., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=75a5a39ee7bf008864556f27870615b9, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:00:32,647 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] ipc.CallRunner(138): callId: 72 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34454 deadline: 1731751292643, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=75a5a39ee7bf008864556f27870615b9, server=037c58a19ddb,37267,1731751077844 2024-11-16T10:00:32,647 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731751217321.75a5a39ee7bf008864556f27870615b9., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 because the exception is null or not the one we care about 2024-11-16T10:00:32,647 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=75a5a39ee7bf008864556f27870615b9, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T10:00:32,647 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731751217321.75a5a39ee7bf008864556f27870615b9., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 , the old value is region=TestAcidGuarantees,,1731751217321.75a5a39ee7bf008864556f27870615b9., hostname=037c58a19ddb,37267,1731751077844, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=75a5a39ee7bf008864556f27870615b9, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:00:32,647 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] ipc.CallRunner(138): callId: 79 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34412 deadline: 1731751292643, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=75a5a39ee7bf008864556f27870615b9, server=037c58a19ddb,37267,1731751077844 2024-11-16T10:00:32,647 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731751217321.75a5a39ee7bf008864556f27870615b9., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=75a5a39ee7bf008864556f27870615b9, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:00:32,647 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731751217321.75a5a39ee7bf008864556f27870615b9., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 because the exception is null or not the one we care about 2024-11-16T10:00:32,647 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731751217321.75a5a39ee7bf008864556f27870615b9., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 , the old value is region=TestAcidGuarantees,,1731751217321.75a5a39ee7bf008864556f27870615b9., hostname=037c58a19ddb,37267,1731751077844, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=75a5a39ee7bf008864556f27870615b9, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:00:32,647 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731751217321.75a5a39ee7bf008864556f27870615b9., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=75a5a39ee7bf008864556f27870615b9, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:00:32,647 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731751217321.75a5a39ee7bf008864556f27870615b9., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 because the exception is null or not the one we care about 2024-11-16T10:00:32,647 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731751217321.75a5a39ee7bf008864556f27870615b9., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 , the old value is region=TestAcidGuarantees,,1731751217321.75a5a39ee7bf008864556f27870615b9., hostname=037c58a19ddb,37267,1731751077844, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=75a5a39ee7bf008864556f27870615b9, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:00:32,647 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731751217321.75a5a39ee7bf008864556f27870615b9., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=75a5a39ee7bf008864556f27870615b9, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:00:32,647 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731751217321.75a5a39ee7bf008864556f27870615b9., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 because the exception is null or not the one we care about 2024-11-16T10:00:32,681 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=37267 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=161 2024-11-16T10:00:32,682 DEBUG [RS_FLUSH_OPERATIONS-regionserver/037c58a19ddb:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731751217321.75a5a39ee7bf008864556f27870615b9. 2024-11-16T10:00:32,682 DEBUG [RS_FLUSH_OPERATIONS-regionserver/037c58a19ddb:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731751217321.75a5a39ee7bf008864556f27870615b9. as already flushing 2024-11-16T10:00:32,682 DEBUG [RS_FLUSH_OPERATIONS-regionserver/037c58a19ddb:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731751217321.75a5a39ee7bf008864556f27870615b9. 2024-11-16T10:00:32,682 ERROR [RS_FLUSH_OPERATIONS-regionserver/037c58a19ddb:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] handler.RSProcedureHandler(58): pid=161 java.io.IOException: Unable to complete flush {ENCODED => 75a5a39ee7bf008864556f27870615b9, NAME => 'TestAcidGuarantees,,1731751217321.75a5a39ee7bf008864556f27870615b9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T10:00:32,682 DEBUG [RS_FLUSH_OPERATIONS-regionserver/037c58a19ddb:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=161 java.io.IOException: Unable to complete flush {ENCODED => 75a5a39ee7bf008864556f27870615b9, NAME => 'TestAcidGuarantees,,1731751217321.75a5a39ee7bf008864556f27870615b9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T10:00:32,682 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37877 {}] master.HMaster(4177): Remote procedure failed, pid=161 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 75a5a39ee7bf008864556f27870615b9, NAME => 'TestAcidGuarantees,,1731751217321.75a5a39ee7bf008864556f27870615b9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 75a5a39ee7bf008864556f27870615b9, NAME => 'TestAcidGuarantees,,1731751217321.75a5a39ee7bf008864556f27870615b9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T10:00:32,693 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37877 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=160 2024-11-16T10:00:32,834 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=37267 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=161 2024-11-16T10:00:32,834 DEBUG [RS_FLUSH_OPERATIONS-regionserver/037c58a19ddb:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731751217321.75a5a39ee7bf008864556f27870615b9. 2024-11-16T10:00:32,834 DEBUG [RS_FLUSH_OPERATIONS-regionserver/037c58a19ddb:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731751217321.75a5a39ee7bf008864556f27870615b9. as already flushing 2024-11-16T10:00:32,834 DEBUG [RS_FLUSH_OPERATIONS-regionserver/037c58a19ddb:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731751217321.75a5a39ee7bf008864556f27870615b9. 2024-11-16T10:00:32,835 ERROR [RS_FLUSH_OPERATIONS-regionserver/037c58a19ddb:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] handler.RSProcedureHandler(58): pid=161 java.io.IOException: Unable to complete flush {ENCODED => 75a5a39ee7bf008864556f27870615b9, NAME => 'TestAcidGuarantees,,1731751217321.75a5a39ee7bf008864556f27870615b9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T10:00:32,835 DEBUG [RS_FLUSH_OPERATIONS-regionserver/037c58a19ddb:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=161 java.io.IOException: Unable to complete flush {ENCODED => 75a5a39ee7bf008864556f27870615b9, NAME => 'TestAcidGuarantees,,1731751217321.75a5a39ee7bf008864556f27870615b9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T10:00:32,835 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37877 {}] master.HMaster(4177): Remote procedure failed, pid=161 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 75a5a39ee7bf008864556f27870615b9, NAME => 'TestAcidGuarantees,,1731751217321.75a5a39ee7bf008864556f27870615b9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 75a5a39ee7bf008864556f27870615b9, NAME => 'TestAcidGuarantees,,1731751217321.75a5a39ee7bf008864556f27870615b9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T10:00:32,855 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=75a5a39ee7bf008864556f27870615b9, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T10:00:32,856 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] ipc.CallRunner(138): callId: 74 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34392 deadline: 1731751292853, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=75a5a39ee7bf008864556f27870615b9, server=037c58a19ddb,37267,1731751077844 2024-11-16T10:00:32,856 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=75a5a39ee7bf008864556f27870615b9, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T10:00:32,856 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34428 deadline: 1731751292853, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=75a5a39ee7bf008864556f27870615b9, server=037c58a19ddb,37267,1731751077844 2024-11-16T10:00:32,856 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731751217321.75a5a39ee7bf008864556f27870615b9., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 , the old value is region=TestAcidGuarantees,,1731751217321.75a5a39ee7bf008864556f27870615b9., hostname=037c58a19ddb,37267,1731751077844, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=75a5a39ee7bf008864556f27870615b9, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:00:32,856 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731751217321.75a5a39ee7bf008864556f27870615b9., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=75a5a39ee7bf008864556f27870615b9, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:00:32,856 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731751217321.75a5a39ee7bf008864556f27870615b9., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 because the exception is null or not the one we care about 2024-11-16T10:00:32,856 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=75a5a39ee7bf008864556f27870615b9, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T10:00:32,856 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731751217321.75a5a39ee7bf008864556f27870615b9., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 , the old value is region=TestAcidGuarantees,,1731751217321.75a5a39ee7bf008864556f27870615b9., hostname=037c58a19ddb,37267,1731751077844, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=75a5a39ee7bf008864556f27870615b9, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:00:32,856 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731751217321.75a5a39ee7bf008864556f27870615b9., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=75a5a39ee7bf008864556f27870615b9, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:00:32,856 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731751217321.75a5a39ee7bf008864556f27870615b9., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 because the exception is null or not the one we care about 2024-11-16T10:00:32,856 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] ipc.CallRunner(138): callId: 76 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34430 deadline: 1731751292853, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=75a5a39ee7bf008864556f27870615b9, server=037c58a19ddb,37267,1731751077844 2024-11-16T10:00:32,856 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=75a5a39ee7bf008864556f27870615b9, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T10:00:32,856 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731751217321.75a5a39ee7bf008864556f27870615b9., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 , the old value is region=TestAcidGuarantees,,1731751217321.75a5a39ee7bf008864556f27870615b9., hostname=037c58a19ddb,37267,1731751077844, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=75a5a39ee7bf008864556f27870615b9, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:00:32,856 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731751217321.75a5a39ee7bf008864556f27870615b9., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=75a5a39ee7bf008864556f27870615b9, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:00:32,856 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] ipc.CallRunner(138): callId: 73 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34454 deadline: 1731751292853, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=75a5a39ee7bf008864556f27870615b9, server=037c58a19ddb,37267,1731751077844 2024-11-16T10:00:32,856 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731751217321.75a5a39ee7bf008864556f27870615b9., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 because the exception is null or not the one we care about 2024-11-16T10:00:32,857 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=75a5a39ee7bf008864556f27870615b9, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T10:00:32,857 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731751217321.75a5a39ee7bf008864556f27870615b9., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 , the old value is region=TestAcidGuarantees,,1731751217321.75a5a39ee7bf008864556f27870615b9., hostname=037c58a19ddb,37267,1731751077844, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=75a5a39ee7bf008864556f27870615b9, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:00:32,857 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731751217321.75a5a39ee7bf008864556f27870615b9., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=75a5a39ee7bf008864556f27870615b9, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:00:32,857 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731751217321.75a5a39ee7bf008864556f27870615b9., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 because the exception is null or not the one we care about 2024-11-16T10:00:32,857 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] ipc.CallRunner(138): callId: 80 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34412 deadline: 1731751292853, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=75a5a39ee7bf008864556f27870615b9, server=037c58a19ddb,37267,1731751077844 2024-11-16T10:00:32,857 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731751217321.75a5a39ee7bf008864556f27870615b9., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 , the old value is region=TestAcidGuarantees,,1731751217321.75a5a39ee7bf008864556f27870615b9., hostname=037c58a19ddb,37267,1731751077844, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=75a5a39ee7bf008864556f27870615b9, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:00:32,857 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731751217321.75a5a39ee7bf008864556f27870615b9., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=75a5a39ee7bf008864556f27870615b9, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:00:32,857 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731751217321.75a5a39ee7bf008864556f27870615b9., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 because the exception is null or not the one we care about 2024-11-16T10:00:32,939 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:32,942 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241116f3686b6e76ea42dbbf0fd2936f4e0b74_75a5a39ee7bf008864556f27870615b9 to hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241116f3686b6e76ea42dbbf0fd2936f4e0b74_75a5a39ee7bf008864556f27870615b9 2024-11-16T10:00:32,943 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/75a5a39ee7bf008864556f27870615b9/.tmp/A/698550d7a97e47fe8a23c3fa8834d663, store: [table=TestAcidGuarantees family=A region=75a5a39ee7bf008864556f27870615b9] 2024-11-16T10:00:32,944 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/75a5a39ee7bf008864556f27870615b9/.tmp/A/698550d7a97e47fe8a23c3fa8834d663 is 175, key is test_row_0/A:col10/1731751232523/Put/seqid=0 2024-11-16T10:00:32,948 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37591 is added to blk_1073742428_1604 (size=31105) 2024-11-16T10:00:32,987 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=37267 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=161 2024-11-16T10:00:32,987 DEBUG [RS_FLUSH_OPERATIONS-regionserver/037c58a19ddb:0-1 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731751217321.75a5a39ee7bf008864556f27870615b9. 2024-11-16T10:00:32,987 DEBUG [RS_FLUSH_OPERATIONS-regionserver/037c58a19ddb:0-1 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731751217321.75a5a39ee7bf008864556f27870615b9. as already flushing 2024-11-16T10:00:32,987 DEBUG [RS_FLUSH_OPERATIONS-regionserver/037c58a19ddb:0-1 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731751217321.75a5a39ee7bf008864556f27870615b9. 2024-11-16T10:00:32,988 ERROR [RS_FLUSH_OPERATIONS-regionserver/037c58a19ddb:0-1 {event_type=RS_FLUSH_REGIONS, pid=161}] handler.RSProcedureHandler(58): pid=161 java.io.IOException: Unable to complete flush {ENCODED => 75a5a39ee7bf008864556f27870615b9, NAME => 'TestAcidGuarantees,,1731751217321.75a5a39ee7bf008864556f27870615b9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T10:00:32,988 DEBUG [RS_FLUSH_OPERATIONS-regionserver/037c58a19ddb:0-1 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=161 java.io.IOException: Unable to complete flush {ENCODED => 75a5a39ee7bf008864556f27870615b9, NAME => 'TestAcidGuarantees,,1731751217321.75a5a39ee7bf008864556f27870615b9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T10:00:32,988 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37877 {}] master.HMaster(4177): Remote procedure failed, pid=161 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 75a5a39ee7bf008864556f27870615b9, NAME => 'TestAcidGuarantees,,1731751217321.75a5a39ee7bf008864556f27870615b9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 75a5a39ee7bf008864556f27870615b9, NAME => 'TestAcidGuarantees,,1731751217321.75a5a39ee7bf008864556f27870615b9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T10:00:33,003 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37877 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=160 2024-11-16T10:00:33,140 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=37267 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=161 2024-11-16T10:00:33,140 DEBUG [RS_FLUSH_OPERATIONS-regionserver/037c58a19ddb:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731751217321.75a5a39ee7bf008864556f27870615b9. 2024-11-16T10:00:33,140 DEBUG [RS_FLUSH_OPERATIONS-regionserver/037c58a19ddb:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731751217321.75a5a39ee7bf008864556f27870615b9. as already flushing 2024-11-16T10:00:33,140 DEBUG [RS_FLUSH_OPERATIONS-regionserver/037c58a19ddb:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731751217321.75a5a39ee7bf008864556f27870615b9. 2024-11-16T10:00:33,140 ERROR [RS_FLUSH_OPERATIONS-regionserver/037c58a19ddb:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] handler.RSProcedureHandler(58): pid=161 java.io.IOException: Unable to complete flush {ENCODED => 75a5a39ee7bf008864556f27870615b9, NAME => 'TestAcidGuarantees,,1731751217321.75a5a39ee7bf008864556f27870615b9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T10:00:33,141 DEBUG [RS_FLUSH_OPERATIONS-regionserver/037c58a19ddb:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=161 java.io.IOException: Unable to complete flush {ENCODED => 75a5a39ee7bf008864556f27870615b9, NAME => 'TestAcidGuarantees,,1731751217321.75a5a39ee7bf008864556f27870615b9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T10:00:33,141 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37877 {}] master.HMaster(4177): Remote procedure failed, pid=161 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 75a5a39ee7bf008864556f27870615b9, NAME => 'TestAcidGuarantees,,1731751217321.75a5a39ee7bf008864556f27870615b9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 75a5a39ee7bf008864556f27870615b9, NAME => 'TestAcidGuarantees,,1731751217321.75a5a39ee7bf008864556f27870615b9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T10:00:33,164 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=75a5a39ee7bf008864556f27870615b9, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T10:00:33,164 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] ipc.CallRunner(138): callId: 75 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34392 deadline: 1731751293162, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=75a5a39ee7bf008864556f27870615b9, server=037c58a19ddb,37267,1731751077844 2024-11-16T10:00:33,165 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=75a5a39ee7bf008864556f27870615b9, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T10:00:33,165 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34428 deadline: 1731751293163, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=75a5a39ee7bf008864556f27870615b9, server=037c58a19ddb,37267,1731751077844 2024-11-16T10:00:33,165 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731751217321.75a5a39ee7bf008864556f27870615b9., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 , the old value is region=TestAcidGuarantees,,1731751217321.75a5a39ee7bf008864556f27870615b9., hostname=037c58a19ddb,37267,1731751077844, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=75a5a39ee7bf008864556f27870615b9, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:00:33,165 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731751217321.75a5a39ee7bf008864556f27870615b9., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=75a5a39ee7bf008864556f27870615b9, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:00:33,165 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731751217321.75a5a39ee7bf008864556f27870615b9., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 because the exception is null or not the one we care about 2024-11-16T10:00:33,165 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731751217321.75a5a39ee7bf008864556f27870615b9., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 , the old value is region=TestAcidGuarantees,,1731751217321.75a5a39ee7bf008864556f27870615b9., hostname=037c58a19ddb,37267,1731751077844, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=75a5a39ee7bf008864556f27870615b9, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:00:33,165 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731751217321.75a5a39ee7bf008864556f27870615b9., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=75a5a39ee7bf008864556f27870615b9, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:00:33,165 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731751217321.75a5a39ee7bf008864556f27870615b9., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 because the exception is null or not the one we care about 2024-11-16T10:00:33,165 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=75a5a39ee7bf008864556f27870615b9, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T10:00:33,165 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] ipc.CallRunner(138): callId: 74 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34454 deadline: 1731751293163, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=75a5a39ee7bf008864556f27870615b9, server=037c58a19ddb,37267,1731751077844 2024-11-16T10:00:33,166 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=75a5a39ee7bf008864556f27870615b9, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T10:00:33,166 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731751217321.75a5a39ee7bf008864556f27870615b9., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 , the old value is region=TestAcidGuarantees,,1731751217321.75a5a39ee7bf008864556f27870615b9., hostname=037c58a19ddb,37267,1731751077844, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=75a5a39ee7bf008864556f27870615b9, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:00:33,166 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] ipc.CallRunner(138): callId: 81 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34412 deadline: 1731751293163, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=75a5a39ee7bf008864556f27870615b9, server=037c58a19ddb,37267,1731751077844 2024-11-16T10:00:33,166 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731751217321.75a5a39ee7bf008864556f27870615b9., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=75a5a39ee7bf008864556f27870615b9, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:00:33,166 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731751217321.75a5a39ee7bf008864556f27870615b9., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 because the exception is null or not the one we care about 2024-11-16T10:00:33,166 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=75a5a39ee7bf008864556f27870615b9, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T10:00:33,166 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731751217321.75a5a39ee7bf008864556f27870615b9., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 , the old value is region=TestAcidGuarantees,,1731751217321.75a5a39ee7bf008864556f27870615b9., hostname=037c58a19ddb,37267,1731751077844, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=75a5a39ee7bf008864556f27870615b9, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:00:33,166 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] ipc.CallRunner(138): callId: 77 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34430 deadline: 1731751293163, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=75a5a39ee7bf008864556f27870615b9, server=037c58a19ddb,37267,1731751077844 2024-11-16T10:00:33,166 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731751217321.75a5a39ee7bf008864556f27870615b9., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=75a5a39ee7bf008864556f27870615b9, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:00:33,166 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731751217321.75a5a39ee7bf008864556f27870615b9., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 because the exception is null or not the one we care about 2024-11-16T10:00:33,166 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731751217321.75a5a39ee7bf008864556f27870615b9., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 , the old value is region=TestAcidGuarantees,,1731751217321.75a5a39ee7bf008864556f27870615b9., hostname=037c58a19ddb,37267,1731751077844, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=75a5a39ee7bf008864556f27870615b9, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:00:33,166 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731751217321.75a5a39ee7bf008864556f27870615b9., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=75a5a39ee7bf008864556f27870615b9, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:00:33,166 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731751217321.75a5a39ee7bf008864556f27870615b9., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 because the exception is null or not the one we care about 2024-11-16T10:00:33,293 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=37267 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=161 2024-11-16T10:00:33,293 DEBUG [RS_FLUSH_OPERATIONS-regionserver/037c58a19ddb:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731751217321.75a5a39ee7bf008864556f27870615b9. 2024-11-16T10:00:33,294 DEBUG [RS_FLUSH_OPERATIONS-regionserver/037c58a19ddb:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731751217321.75a5a39ee7bf008864556f27870615b9. as already flushing 2024-11-16T10:00:33,294 DEBUG [RS_FLUSH_OPERATIONS-regionserver/037c58a19ddb:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731751217321.75a5a39ee7bf008864556f27870615b9. 2024-11-16T10:00:33,294 ERROR [RS_FLUSH_OPERATIONS-regionserver/037c58a19ddb:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] handler.RSProcedureHandler(58): pid=161 java.io.IOException: Unable to complete flush {ENCODED => 75a5a39ee7bf008864556f27870615b9, NAME => 'TestAcidGuarantees,,1731751217321.75a5a39ee7bf008864556f27870615b9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T10:00:33,294 DEBUG [RS_FLUSH_OPERATIONS-regionserver/037c58a19ddb:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=161 java.io.IOException: Unable to complete flush {ENCODED => 75a5a39ee7bf008864556f27870615b9, NAME => 'TestAcidGuarantees,,1731751217321.75a5a39ee7bf008864556f27870615b9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T10:00:33,295 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37877 {}] master.HMaster(4177): Remote procedure failed, pid=161 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 75a5a39ee7bf008864556f27870615b9, NAME => 'TestAcidGuarantees,,1731751217321.75a5a39ee7bf008864556f27870615b9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 75a5a39ee7bf008864556f27870615b9, NAME => 'TestAcidGuarantees,,1731751217321.75a5a39ee7bf008864556f27870615b9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T10:00:33,348 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=238, memsize=55.9 K, hasBloomFilter=true, into tmp file hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/75a5a39ee7bf008864556f27870615b9/.tmp/A/698550d7a97e47fe8a23c3fa8834d663 2024-11-16T10:00:33,356 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/75a5a39ee7bf008864556f27870615b9/.tmp/B/518123bb8da8418cb9cb610105faac82 is 50, key is test_row_0/B:col10/1731751232523/Put/seqid=0 2024-11-16T10:00:33,359 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37591 is added to blk_1073742429_1605 (size=12151) 2024-11-16T10:00:33,446 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=37267 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=161 2024-11-16T10:00:33,446 DEBUG [RS_FLUSH_OPERATIONS-regionserver/037c58a19ddb:0-1 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731751217321.75a5a39ee7bf008864556f27870615b9. 2024-11-16T10:00:33,446 DEBUG [RS_FLUSH_OPERATIONS-regionserver/037c58a19ddb:0-1 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731751217321.75a5a39ee7bf008864556f27870615b9. as already flushing 2024-11-16T10:00:33,446 DEBUG [RS_FLUSH_OPERATIONS-regionserver/037c58a19ddb:0-1 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731751217321.75a5a39ee7bf008864556f27870615b9. 2024-11-16T10:00:33,446 ERROR [RS_FLUSH_OPERATIONS-regionserver/037c58a19ddb:0-1 {event_type=RS_FLUSH_REGIONS, pid=161}] handler.RSProcedureHandler(58): pid=161 java.io.IOException: Unable to complete flush {ENCODED => 75a5a39ee7bf008864556f27870615b9, NAME => 'TestAcidGuarantees,,1731751217321.75a5a39ee7bf008864556f27870615b9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T10:00:33,447 DEBUG [RS_FLUSH_OPERATIONS-regionserver/037c58a19ddb:0-1 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=161 java.io.IOException: Unable to complete flush {ENCODED => 75a5a39ee7bf008864556f27870615b9, NAME => 'TestAcidGuarantees,,1731751217321.75a5a39ee7bf008864556f27870615b9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T10:00:33,447 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37877 {}] master.HMaster(4177): Remote procedure failed, pid=161 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 75a5a39ee7bf008864556f27870615b9, NAME => 'TestAcidGuarantees,,1731751217321.75a5a39ee7bf008864556f27870615b9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 75a5a39ee7bf008864556f27870615b9, NAME => 'TestAcidGuarantees,,1731751217321.75a5a39ee7bf008864556f27870615b9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T10:00:33,513 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37877 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=160 2024-11-16T10:00:33,599 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=37267 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=161 2024-11-16T10:00:33,599 DEBUG [RS_FLUSH_OPERATIONS-regionserver/037c58a19ddb:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731751217321.75a5a39ee7bf008864556f27870615b9. 2024-11-16T10:00:33,599 DEBUG [RS_FLUSH_OPERATIONS-regionserver/037c58a19ddb:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731751217321.75a5a39ee7bf008864556f27870615b9. as already flushing 2024-11-16T10:00:33,599 DEBUG [RS_FLUSH_OPERATIONS-regionserver/037c58a19ddb:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731751217321.75a5a39ee7bf008864556f27870615b9. 2024-11-16T10:00:33,599 ERROR [RS_FLUSH_OPERATIONS-regionserver/037c58a19ddb:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] handler.RSProcedureHandler(58): pid=161 java.io.IOException: Unable to complete flush {ENCODED => 75a5a39ee7bf008864556f27870615b9, NAME => 'TestAcidGuarantees,,1731751217321.75a5a39ee7bf008864556f27870615b9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T10:00:33,599 DEBUG [RS_FLUSH_OPERATIONS-regionserver/037c58a19ddb:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=161 java.io.IOException: Unable to complete flush {ENCODED => 75a5a39ee7bf008864556f27870615b9, NAME => 'TestAcidGuarantees,,1731751217321.75a5a39ee7bf008864556f27870615b9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T10:00:33,600 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37877 {}] master.HMaster(4177): Remote procedure failed, pid=161 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 75a5a39ee7bf008864556f27870615b9, NAME => 'TestAcidGuarantees,,1731751217321.75a5a39ee7bf008864556f27870615b9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 75a5a39ee7bf008864556f27870615b9, NAME => 'TestAcidGuarantees,,1731751217321.75a5a39ee7bf008864556f27870615b9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T10:00:33,675 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=75a5a39ee7bf008864556f27870615b9, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T10:00:33,675 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] ipc.CallRunner(138): callId: 76 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34392 deadline: 1731751293673, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=75a5a39ee7bf008864556f27870615b9, server=037c58a19ddb,37267,1731751077844 2024-11-16T10:00:33,675 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=75a5a39ee7bf008864556f27870615b9, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T10:00:33,676 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731751217321.75a5a39ee7bf008864556f27870615b9., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 , the old value is region=TestAcidGuarantees,,1731751217321.75a5a39ee7bf008864556f27870615b9., hostname=037c58a19ddb,37267,1731751077844, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=75a5a39ee7bf008864556f27870615b9, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:00:33,676 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731751217321.75a5a39ee7bf008864556f27870615b9., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=75a5a39ee7bf008864556f27870615b9, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:00:33,676 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34428 deadline: 1731751293673, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=75a5a39ee7bf008864556f27870615b9, server=037c58a19ddb,37267,1731751077844 2024-11-16T10:00:33,676 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731751217321.75a5a39ee7bf008864556f27870615b9., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 because the exception is null or not the one we care about 2024-11-16T10:00:33,676 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=75a5a39ee7bf008864556f27870615b9, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T10:00:33,676 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731751217321.75a5a39ee7bf008864556f27870615b9., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 , the old value is region=TestAcidGuarantees,,1731751217321.75a5a39ee7bf008864556f27870615b9., hostname=037c58a19ddb,37267,1731751077844, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=75a5a39ee7bf008864556f27870615b9, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:00:33,676 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731751217321.75a5a39ee7bf008864556f27870615b9., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=75a5a39ee7bf008864556f27870615b9, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:00:33,676 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] ipc.CallRunner(138): callId: 75 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34454 deadline: 1731751293673, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=75a5a39ee7bf008864556f27870615b9, server=037c58a19ddb,37267,1731751077844 2024-11-16T10:00:33,676 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731751217321.75a5a39ee7bf008864556f27870615b9., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 because the exception is null or not the one we care about 2024-11-16T10:00:33,676 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=75a5a39ee7bf008864556f27870615b9, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T10:00:33,676 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] ipc.CallRunner(138): callId: 82 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34412 deadline: 1731751293673, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=75a5a39ee7bf008864556f27870615b9, server=037c58a19ddb,37267,1731751077844 2024-11-16T10:00:33,676 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731751217321.75a5a39ee7bf008864556f27870615b9., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 , the old value is region=TestAcidGuarantees,,1731751217321.75a5a39ee7bf008864556f27870615b9., hostname=037c58a19ddb,37267,1731751077844, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=75a5a39ee7bf008864556f27870615b9, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:00:33,676 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731751217321.75a5a39ee7bf008864556f27870615b9., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=75a5a39ee7bf008864556f27870615b9, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:00:33,676 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731751217321.75a5a39ee7bf008864556f27870615b9., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 because the exception is null or not the one we care about 2024-11-16T10:00:33,676 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=75a5a39ee7bf008864556f27870615b9, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T10:00:33,676 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731751217321.75a5a39ee7bf008864556f27870615b9., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 , the old value is region=TestAcidGuarantees,,1731751217321.75a5a39ee7bf008864556f27870615b9., hostname=037c58a19ddb,37267,1731751077844, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=75a5a39ee7bf008864556f27870615b9, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:00:33,676 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] ipc.CallRunner(138): callId: 78 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34430 deadline: 1731751293673, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=75a5a39ee7bf008864556f27870615b9, server=037c58a19ddb,37267,1731751077844 2024-11-16T10:00:33,676 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731751217321.75a5a39ee7bf008864556f27870615b9., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=75a5a39ee7bf008864556f27870615b9, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:00:33,676 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731751217321.75a5a39ee7bf008864556f27870615b9., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 because the exception is null or not the one we care about 2024-11-16T10:00:33,677 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731751217321.75a5a39ee7bf008864556f27870615b9., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 , the old value is region=TestAcidGuarantees,,1731751217321.75a5a39ee7bf008864556f27870615b9., hostname=037c58a19ddb,37267,1731751077844, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=75a5a39ee7bf008864556f27870615b9, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:00:33,677 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731751217321.75a5a39ee7bf008864556f27870615b9., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=75a5a39ee7bf008864556f27870615b9, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:00:33,677 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731751217321.75a5a39ee7bf008864556f27870615b9., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 because the exception is null or not the one we care about 2024-11-16T10:00:33,752 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=37267 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=161 2024-11-16T10:00:33,752 DEBUG [RS_FLUSH_OPERATIONS-regionserver/037c58a19ddb:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731751217321.75a5a39ee7bf008864556f27870615b9. 2024-11-16T10:00:33,752 DEBUG [RS_FLUSH_OPERATIONS-regionserver/037c58a19ddb:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731751217321.75a5a39ee7bf008864556f27870615b9. as already flushing 2024-11-16T10:00:33,752 DEBUG [RS_FLUSH_OPERATIONS-regionserver/037c58a19ddb:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731751217321.75a5a39ee7bf008864556f27870615b9. 2024-11-16T10:00:33,752 ERROR [RS_FLUSH_OPERATIONS-regionserver/037c58a19ddb:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] handler.RSProcedureHandler(58): pid=161 java.io.IOException: Unable to complete flush {ENCODED => 75a5a39ee7bf008864556f27870615b9, NAME => 'TestAcidGuarantees,,1731751217321.75a5a39ee7bf008864556f27870615b9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T10:00:33,752 DEBUG [RS_FLUSH_OPERATIONS-regionserver/037c58a19ddb:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=161 java.io.IOException: Unable to complete flush {ENCODED => 75a5a39ee7bf008864556f27870615b9, NAME => 'TestAcidGuarantees,,1731751217321.75a5a39ee7bf008864556f27870615b9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T10:00:33,753 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37877 {}] master.HMaster(4177): Remote procedure failed, pid=161 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 75a5a39ee7bf008864556f27870615b9, NAME => 'TestAcidGuarantees,,1731751217321.75a5a39ee7bf008864556f27870615b9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 75a5a39ee7bf008864556f27870615b9, NAME => 'TestAcidGuarantees,,1731751217321.75a5a39ee7bf008864556f27870615b9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T10:00:33,759 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=55.91 KB at sequenceid=238 (bloomFilter=true), to=hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/75a5a39ee7bf008864556f27870615b9/.tmp/B/518123bb8da8418cb9cb610105faac82 2024-11-16T10:00:33,775 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/75a5a39ee7bf008864556f27870615b9/.tmp/C/26e3e62e9e914fa28e8cff1840ba7691 is 50, key is test_row_0/C:col10/1731751232523/Put/seqid=0 2024-11-16T10:00:33,778 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37591 is added to blk_1073742430_1606 (size=12151) 2024-11-16T10:00:33,904 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=37267 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=161 2024-11-16T10:00:33,905 DEBUG [RS_FLUSH_OPERATIONS-regionserver/037c58a19ddb:0-1 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731751217321.75a5a39ee7bf008864556f27870615b9. 2024-11-16T10:00:33,905 DEBUG [RS_FLUSH_OPERATIONS-regionserver/037c58a19ddb:0-1 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731751217321.75a5a39ee7bf008864556f27870615b9. as already flushing 2024-11-16T10:00:33,905 DEBUG [RS_FLUSH_OPERATIONS-regionserver/037c58a19ddb:0-1 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731751217321.75a5a39ee7bf008864556f27870615b9. 2024-11-16T10:00:33,905 ERROR [RS_FLUSH_OPERATIONS-regionserver/037c58a19ddb:0-1 {event_type=RS_FLUSH_REGIONS, pid=161}] handler.RSProcedureHandler(58): pid=161 java.io.IOException: Unable to complete flush {ENCODED => 75a5a39ee7bf008864556f27870615b9, NAME => 'TestAcidGuarantees,,1731751217321.75a5a39ee7bf008864556f27870615b9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T10:00:33,905 DEBUG [RS_FLUSH_OPERATIONS-regionserver/037c58a19ddb:0-1 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=161 java.io.IOException: Unable to complete flush {ENCODED => 75a5a39ee7bf008864556f27870615b9, NAME => 'TestAcidGuarantees,,1731751217321.75a5a39ee7bf008864556f27870615b9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T10:00:33,906 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37877 {}] master.HMaster(4177): Remote procedure failed, pid=161 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 75a5a39ee7bf008864556f27870615b9, NAME => 'TestAcidGuarantees,,1731751217321.75a5a39ee7bf008864556f27870615b9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 75a5a39ee7bf008864556f27870615b9, NAME => 'TestAcidGuarantees,,1731751217321.75a5a39ee7bf008864556f27870615b9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T10:00:34,057 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=37267 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=161 2024-11-16T10:00:34,058 DEBUG [RS_FLUSH_OPERATIONS-regionserver/037c58a19ddb:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731751217321.75a5a39ee7bf008864556f27870615b9. 2024-11-16T10:00:34,058 DEBUG [RS_FLUSH_OPERATIONS-regionserver/037c58a19ddb:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731751217321.75a5a39ee7bf008864556f27870615b9. as already flushing 2024-11-16T10:00:34,058 DEBUG [RS_FLUSH_OPERATIONS-regionserver/037c58a19ddb:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731751217321.75a5a39ee7bf008864556f27870615b9. 2024-11-16T10:00:34,058 ERROR [RS_FLUSH_OPERATIONS-regionserver/037c58a19ddb:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] handler.RSProcedureHandler(58): pid=161 java.io.IOException: Unable to complete flush {ENCODED => 75a5a39ee7bf008864556f27870615b9, NAME => 'TestAcidGuarantees,,1731751217321.75a5a39ee7bf008864556f27870615b9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T10:00:34,058 DEBUG [RS_FLUSH_OPERATIONS-regionserver/037c58a19ddb:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=161 java.io.IOException: Unable to complete flush {ENCODED => 75a5a39ee7bf008864556f27870615b9, NAME => 'TestAcidGuarantees,,1731751217321.75a5a39ee7bf008864556f27870615b9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T10:00:34,058 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37877 {}] master.HMaster(4177): Remote procedure failed, pid=161 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 75a5a39ee7bf008864556f27870615b9, NAME => 'TestAcidGuarantees,,1731751217321.75a5a39ee7bf008864556f27870615b9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 75a5a39ee7bf008864556f27870615b9, NAME => 'TestAcidGuarantees,,1731751217321.75a5a39ee7bf008864556f27870615b9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T10:00:34,179 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=55.91 KB at sequenceid=238 (bloomFilter=true), to=hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/75a5a39ee7bf008864556f27870615b9/.tmp/C/26e3e62e9e914fa28e8cff1840ba7691 2024-11-16T10:00:34,184 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/75a5a39ee7bf008864556f27870615b9/.tmp/A/698550d7a97e47fe8a23c3fa8834d663 as hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/75a5a39ee7bf008864556f27870615b9/A/698550d7a97e47fe8a23c3fa8834d663 2024-11-16T10:00:34,187 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/75a5a39ee7bf008864556f27870615b9/A/698550d7a97e47fe8a23c3fa8834d663, entries=150, sequenceid=238, filesize=30.4 K 2024-11-16T10:00:34,188 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/75a5a39ee7bf008864556f27870615b9/.tmp/B/518123bb8da8418cb9cb610105faac82 as hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/75a5a39ee7bf008864556f27870615b9/B/518123bb8da8418cb9cb610105faac82 2024-11-16T10:00:34,192 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/75a5a39ee7bf008864556f27870615b9/B/518123bb8da8418cb9cb610105faac82, entries=150, sequenceid=238, filesize=11.9 K 2024-11-16T10:00:34,192 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/75a5a39ee7bf008864556f27870615b9/.tmp/C/26e3e62e9e914fa28e8cff1840ba7691 as hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/75a5a39ee7bf008864556f27870615b9/C/26e3e62e9e914fa28e8cff1840ba7691 2024-11-16T10:00:34,195 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/75a5a39ee7bf008864556f27870615b9/C/26e3e62e9e914fa28e8cff1840ba7691, entries=150, sequenceid=238, filesize=11.9 K 2024-11-16T10:00:34,196 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~167.72 KB/171750, heapSize ~440.16 KB/450720, currentSize=33.54 KB/34350 for 75a5a39ee7bf008864556f27870615b9 in 1672ms, sequenceid=238, compaction requested=true 2024-11-16T10:00:34,196 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 75a5a39ee7bf008864556f27870615b9: 2024-11-16T10:00:34,196 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 75a5a39ee7bf008864556f27870615b9:A, priority=-2147483648, current under compaction store size is 1 2024-11-16T10:00:34,196 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-16T10:00:34,196 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 75a5a39ee7bf008864556f27870615b9:B, priority=-2147483648, current under compaction store size is 2 2024-11-16T10:00:34,196 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-16T10:00:34,196 DEBUG [RS:0;037c58a19ddb:37267-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-16T10:00:34,197 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 75a5a39ee7bf008864556f27870615b9:C, priority=-2147483648, current under compaction store size is 3 2024-11-16T10:00:34,197 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-16T10:00:34,197 DEBUG [RS:0;037c58a19ddb:37267-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-16T10:00:34,198 DEBUG [RS:0;037c58a19ddb:37267-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36897 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-16T10:00:34,198 DEBUG [RS:0;037c58a19ddb:37267-longCompactions-0 {}] regionserver.HStore(1541): 75a5a39ee7bf008864556f27870615b9/B is initiating minor compaction (all files) 2024-11-16T10:00:34,198 INFO [RS:0;037c58a19ddb:37267-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 75a5a39ee7bf008864556f27870615b9/B in TestAcidGuarantees,,1731751217321.75a5a39ee7bf008864556f27870615b9. 2024-11-16T10:00:34,198 INFO [RS:0;037c58a19ddb:37267-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/75a5a39ee7bf008864556f27870615b9/B/d4e00a75365b4c6e937dcf823b92da7c, hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/75a5a39ee7bf008864556f27870615b9/B/fdace4a2305248da8f37008895cb2774, hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/75a5a39ee7bf008864556f27870615b9/B/518123bb8da8418cb9cb610105faac82] into tmpdir=hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/75a5a39ee7bf008864556f27870615b9/.tmp, totalSize=36.0 K 2024-11-16T10:00:34,198 DEBUG [RS:0;037c58a19ddb:37267-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 93759 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-16T10:00:34,198 DEBUG [RS:0;037c58a19ddb:37267-shortCompactions-0 {}] regionserver.HStore(1541): 75a5a39ee7bf008864556f27870615b9/A is initiating minor compaction (all files) 2024-11-16T10:00:34,198 DEBUG [RS:0;037c58a19ddb:37267-longCompactions-0 {}] compactions.Compactor(225): Compacting d4e00a75365b4c6e937dcf823b92da7c, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=197, earliestPutTs=1731751229586 2024-11-16T10:00:34,198 INFO [RS:0;037c58a19ddb:37267-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 75a5a39ee7bf008864556f27870615b9/A in TestAcidGuarantees,,1731751217321.75a5a39ee7bf008864556f27870615b9. 2024-11-16T10:00:34,198 INFO [RS:0;037c58a19ddb:37267-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/75a5a39ee7bf008864556f27870615b9/A/b9bdffdf7c5d49568463afe11efd7753, hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/75a5a39ee7bf008864556f27870615b9/A/584e3cb61e0b4e15acdabe73ca47b987, hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/75a5a39ee7bf008864556f27870615b9/A/698550d7a97e47fe8a23c3fa8834d663] into tmpdir=hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/75a5a39ee7bf008864556f27870615b9/.tmp, totalSize=91.6 K 2024-11-16T10:00:34,198 INFO [RS:0;037c58a19ddb:37267-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731751217321.75a5a39ee7bf008864556f27870615b9. 2024-11-16T10:00:34,198 DEBUG [RS:0;037c58a19ddb:37267-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731751217321.75a5a39ee7bf008864556f27870615b9. files: [hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/75a5a39ee7bf008864556f27870615b9/A/b9bdffdf7c5d49568463afe11efd7753, hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/75a5a39ee7bf008864556f27870615b9/A/584e3cb61e0b4e15acdabe73ca47b987, hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/75a5a39ee7bf008864556f27870615b9/A/698550d7a97e47fe8a23c3fa8834d663] 2024-11-16T10:00:34,199 DEBUG [RS:0;037c58a19ddb:37267-longCompactions-0 {}] compactions.Compactor(225): Compacting fdace4a2305248da8f37008895cb2774, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=209, earliestPutTs=1731751229705 2024-11-16T10:00:34,199 DEBUG [RS:0;037c58a19ddb:37267-shortCompactions-0 {}] compactions.Compactor(225): Compacting b9bdffdf7c5d49568463afe11efd7753, keycount=150, bloomtype=ROW, size=30.8 K, encoding=NONE, compression=NONE, seqNum=197, earliestPutTs=1731751229586 2024-11-16T10:00:34,199 DEBUG [RS:0;037c58a19ddb:37267-longCompactions-0 {}] compactions.Compactor(225): Compacting 518123bb8da8418cb9cb610105faac82, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=238, earliestPutTs=1731751231889 2024-11-16T10:00:34,199 DEBUG [RS:0;037c58a19ddb:37267-shortCompactions-0 {}] compactions.Compactor(225): Compacting 584e3cb61e0b4e15acdabe73ca47b987, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=209, earliestPutTs=1731751229705 2024-11-16T10:00:34,199 DEBUG [RS:0;037c58a19ddb:37267-shortCompactions-0 {}] compactions.Compactor(225): Compacting 698550d7a97e47fe8a23c3fa8834d663, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=238, earliestPutTs=1731751231889 2024-11-16T10:00:34,205 INFO [RS:0;037c58a19ddb:37267-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 75a5a39ee7bf008864556f27870615b9#B#compaction#514 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-16T10:00:34,206 DEBUG [RS:0;037c58a19ddb:37267-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/75a5a39ee7bf008864556f27870615b9/.tmp/B/2dac6b2aba484c8ca3588f74e5a36789 is 50, key is test_row_0/B:col10/1731751232523/Put/seqid=0 2024-11-16T10:00:34,207 INFO [RS:0;037c58a19ddb:37267-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=75a5a39ee7bf008864556f27870615b9] 2024-11-16T10:00:34,209 DEBUG [RS:0;037c58a19ddb:37267-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241116fc98d4081fea4cd4bdf3a23f26ff6d60_75a5a39ee7bf008864556f27870615b9 store=[table=TestAcidGuarantees family=A region=75a5a39ee7bf008864556f27870615b9] 2024-11-16T10:00:34,210 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=37267 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=161 2024-11-16T10:00:34,210 DEBUG [RS_FLUSH_OPERATIONS-regionserver/037c58a19ddb:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731751217321.75a5a39ee7bf008864556f27870615b9. 2024-11-16T10:00:34,210 INFO [RS_FLUSH_OPERATIONS-regionserver/037c58a19ddb:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.HRegion(2902): Flushing 75a5a39ee7bf008864556f27870615b9 3/3 column families, dataSize=33.54 KB heapSize=88.64 KB 2024-11-16T10:00:34,210 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37591 is added to blk_1073742431_1607 (size=12697) 2024-11-16T10:00:34,211 DEBUG [RS:0;037c58a19ddb:37267-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241116fc98d4081fea4cd4bdf3a23f26ff6d60_75a5a39ee7bf008864556f27870615b9, store=[table=TestAcidGuarantees family=A region=75a5a39ee7bf008864556f27870615b9] 2024-11-16T10:00:34,211 DEBUG [RS:0;037c58a19ddb:37267-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241116fc98d4081fea4cd4bdf3a23f26ff6d60_75a5a39ee7bf008864556f27870615b9 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=75a5a39ee7bf008864556f27870615b9] 2024-11-16T10:00:34,216 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37591 is added to blk_1073742432_1608 (size=4469) 2024-11-16T10:00:34,218 INFO [RS:0;037c58a19ddb:37267-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 75a5a39ee7bf008864556f27870615b9#A#compaction#515 average throughput is 2.44 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-16T10:00:34,218 DEBUG [RS:0;037c58a19ddb:37267-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/75a5a39ee7bf008864556f27870615b9/.tmp/A/f234c94f31c345bdacdbdad119738a8c is 175, key is test_row_0/A:col10/1731751232523/Put/seqid=0 2024-11-16T10:00:34,219 DEBUG [RS_FLUSH_OPERATIONS-regionserver/037c58a19ddb:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411165f9e28dbe5ae434d9a6b4b81a75e72c8_75a5a39ee7bf008864556f27870615b9 is 50, key is test_row_0/A:col10/1731751232527/Put/seqid=0 2024-11-16T10:00:34,220 DEBUG [RS:0;037c58a19ddb:37267-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/75a5a39ee7bf008864556f27870615b9/.tmp/B/2dac6b2aba484c8ca3588f74e5a36789 as hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/75a5a39ee7bf008864556f27870615b9/B/2dac6b2aba484c8ca3588f74e5a36789 2024-11-16T10:00:34,224 INFO [RS:0;037c58a19ddb:37267-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 75a5a39ee7bf008864556f27870615b9/B of 75a5a39ee7bf008864556f27870615b9 into 2dac6b2aba484c8ca3588f74e5a36789(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-16T10:00:34,224 DEBUG [RS:0;037c58a19ddb:37267-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 75a5a39ee7bf008864556f27870615b9: 2024-11-16T10:00:34,224 INFO [RS:0;037c58a19ddb:37267-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731751217321.75a5a39ee7bf008864556f27870615b9., storeName=75a5a39ee7bf008864556f27870615b9/B, priority=13, startTime=1731751234196; duration=0sec 2024-11-16T10:00:34,224 DEBUG [RS:0;037c58a19ddb:37267-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-16T10:00:34,224 DEBUG [RS:0;037c58a19ddb:37267-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 75a5a39ee7bf008864556f27870615b9:B 2024-11-16T10:00:34,224 DEBUG [RS:0;037c58a19ddb:37267-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-16T10:00:34,225 DEBUG [RS:0;037c58a19ddb:37267-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36897 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-16T10:00:34,225 DEBUG [RS:0;037c58a19ddb:37267-longCompactions-0 {}] regionserver.HStore(1541): 75a5a39ee7bf008864556f27870615b9/C is initiating minor compaction (all files) 2024-11-16T10:00:34,225 INFO [RS:0;037c58a19ddb:37267-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 75a5a39ee7bf008864556f27870615b9/C in TestAcidGuarantees,,1731751217321.75a5a39ee7bf008864556f27870615b9. 2024-11-16T10:00:34,225 INFO [RS:0;037c58a19ddb:37267-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/75a5a39ee7bf008864556f27870615b9/C/bb43f99d4a98465cbfa41dff2bac86f9, hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/75a5a39ee7bf008864556f27870615b9/C/39ac7213ec73453dabe0a6d1fbad50a7, hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/75a5a39ee7bf008864556f27870615b9/C/26e3e62e9e914fa28e8cff1840ba7691] into tmpdir=hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/75a5a39ee7bf008864556f27870615b9/.tmp, totalSize=36.0 K 2024-11-16T10:00:34,226 DEBUG [RS:0;037c58a19ddb:37267-longCompactions-0 {}] compactions.Compactor(225): Compacting bb43f99d4a98465cbfa41dff2bac86f9, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=197, earliestPutTs=1731751229586 2024-11-16T10:00:34,226 DEBUG [RS:0;037c58a19ddb:37267-longCompactions-0 {}] compactions.Compactor(225): Compacting 39ac7213ec73453dabe0a6d1fbad50a7, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=209, earliestPutTs=1731751229705 2024-11-16T10:00:34,227 DEBUG [RS:0;037c58a19ddb:37267-longCompactions-0 {}] compactions.Compactor(225): Compacting 26e3e62e9e914fa28e8cff1840ba7691, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=238, earliestPutTs=1731751231889 2024-11-16T10:00:34,237 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37591 is added to blk_1073742433_1609 (size=31651) 2024-11-16T10:00:34,249 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37591 is added to blk_1073742434_1610 (size=12304) 2024-11-16T10:00:34,250 DEBUG [RS_FLUSH_OPERATIONS-regionserver/037c58a19ddb:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:34,251 INFO [RS:0;037c58a19ddb:37267-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 75a5a39ee7bf008864556f27870615b9#C#compaction#517 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-16T10:00:34,251 DEBUG [RS:0;037c58a19ddb:37267-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/75a5a39ee7bf008864556f27870615b9/.tmp/C/c252c96455d64fc29b7b1cda3f9f2f13 is 50, key is test_row_0/C:col10/1731751232523/Put/seqid=0 2024-11-16T10:00:34,253 INFO [RS_FLUSH_OPERATIONS-regionserver/037c58a19ddb:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411165f9e28dbe5ae434d9a6b4b81a75e72c8_75a5a39ee7bf008864556f27870615b9 to hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411165f9e28dbe5ae434d9a6b4b81a75e72c8_75a5a39ee7bf008864556f27870615b9 2024-11-16T10:00:34,254 DEBUG [RS_FLUSH_OPERATIONS-regionserver/037c58a19ddb:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/75a5a39ee7bf008864556f27870615b9/.tmp/A/a15a55fc49b74b35997765299aa047dc, store: [table=TestAcidGuarantees family=A region=75a5a39ee7bf008864556f27870615b9] 2024-11-16T10:00:34,255 DEBUG [RS_FLUSH_OPERATIONS-regionserver/037c58a19ddb:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/75a5a39ee7bf008864556f27870615b9/.tmp/A/a15a55fc49b74b35997765299aa047dc is 175, key is test_row_0/A:col10/1731751232527/Put/seqid=0 2024-11-16T10:00:34,256 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37591 is added to blk_1073742435_1611 (size=12697) 2024-11-16T10:00:34,259 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37591 is added to blk_1073742436_1612 (size=31105) 2024-11-16T10:00:34,259 INFO [RS_FLUSH_OPERATIONS-regionserver/037c58a19ddb:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=246, memsize=11.2 K, hasBloomFilter=true, into tmp file hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/75a5a39ee7bf008864556f27870615b9/.tmp/A/a15a55fc49b74b35997765299aa047dc 2024-11-16T10:00:34,265 DEBUG [RS_FLUSH_OPERATIONS-regionserver/037c58a19ddb:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/75a5a39ee7bf008864556f27870615b9/.tmp/B/e59fe4cb81fc403999f5bb2b146958fc is 50, key is test_row_0/B:col10/1731751232527/Put/seqid=0 2024-11-16T10:00:34,270 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37591 is added to blk_1073742437_1613 (size=12151) 2024-11-16T10:00:34,522 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37877 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=160 2024-11-16T10:00:34,642 DEBUG [RS:0;037c58a19ddb:37267-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/75a5a39ee7bf008864556f27870615b9/.tmp/A/f234c94f31c345bdacdbdad119738a8c as hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/75a5a39ee7bf008864556f27870615b9/A/f234c94f31c345bdacdbdad119738a8c 2024-11-16T10:00:34,647 INFO [RS:0;037c58a19ddb:37267-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 75a5a39ee7bf008864556f27870615b9/A of 75a5a39ee7bf008864556f27870615b9 into f234c94f31c345bdacdbdad119738a8c(size=30.9 K), total size for store is 30.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-16T10:00:34,647 DEBUG [RS:0;037c58a19ddb:37267-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 75a5a39ee7bf008864556f27870615b9: 2024-11-16T10:00:34,647 INFO [RS:0;037c58a19ddb:37267-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731751217321.75a5a39ee7bf008864556f27870615b9., storeName=75a5a39ee7bf008864556f27870615b9/A, priority=13, startTime=1731751234196; duration=0sec 2024-11-16T10:00:34,647 DEBUG [RS:0;037c58a19ddb:37267-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-16T10:00:34,647 DEBUG [RS:0;037c58a19ddb:37267-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 75a5a39ee7bf008864556f27870615b9:A 2024-11-16T10:00:34,660 DEBUG [RS:0;037c58a19ddb:37267-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/75a5a39ee7bf008864556f27870615b9/.tmp/C/c252c96455d64fc29b7b1cda3f9f2f13 as hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/75a5a39ee7bf008864556f27870615b9/C/c252c96455d64fc29b7b1cda3f9f2f13 2024-11-16T10:00:34,664 INFO [RS:0;037c58a19ddb:37267-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 75a5a39ee7bf008864556f27870615b9/C of 75a5a39ee7bf008864556f27870615b9 into c252c96455d64fc29b7b1cda3f9f2f13(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-16T10:00:34,665 DEBUG [RS:0;037c58a19ddb:37267-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 75a5a39ee7bf008864556f27870615b9: 2024-11-16T10:00:34,665 INFO [RS:0;037c58a19ddb:37267-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731751217321.75a5a39ee7bf008864556f27870615b9., storeName=75a5a39ee7bf008864556f27870615b9/C, priority=13, startTime=1731751234196; duration=0sec 2024-11-16T10:00:34,665 DEBUG [RS:0;037c58a19ddb:37267-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-16T10:00:34,665 DEBUG [RS:0;037c58a19ddb:37267-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 75a5a39ee7bf008864556f27870615b9:C 2024-11-16T10:00:34,671 INFO [RS_FLUSH_OPERATIONS-regionserver/037c58a19ddb:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=246 (bloomFilter=true), to=hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/75a5a39ee7bf008864556f27870615b9/.tmp/B/e59fe4cb81fc403999f5bb2b146958fc 2024-11-16T10:00:34,677 DEBUG [RS_FLUSH_OPERATIONS-regionserver/037c58a19ddb:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/75a5a39ee7bf008864556f27870615b9/.tmp/C/f158f302c9e4447abd35ad6d46fa5b3e is 50, key is test_row_0/C:col10/1731751232527/Put/seqid=0 2024-11-16T10:00:34,685 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731751217321.75a5a39ee7bf008864556f27870615b9. as already flushing 2024-11-16T10:00:34,685 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] regionserver.HRegion(8855): Flush requested on 75a5a39ee7bf008864556f27870615b9 2024-11-16T10:00:34,692 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37591 is added to blk_1073742438_1614 (size=12151) 2024-11-16T10:00:34,717 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=75a5a39ee7bf008864556f27870615b9, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T10:00:34,718 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] ipc.CallRunner(138): callId: 84 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34430 deadline: 1731751294713, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=75a5a39ee7bf008864556f27870615b9, server=037c58a19ddb,37267,1731751077844 2024-11-16T10:00:34,718 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=75a5a39ee7bf008864556f27870615b9, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T10:00:34,718 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731751217321.75a5a39ee7bf008864556f27870615b9., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 , the old value is region=TestAcidGuarantees,,1731751217321.75a5a39ee7bf008864556f27870615b9., hostname=037c58a19ddb,37267,1731751077844, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=75a5a39ee7bf008864556f27870615b9, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:00:34,718 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] ipc.CallRunner(138): callId: 81 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34454 deadline: 1731751294714, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=75a5a39ee7bf008864556f27870615b9, server=037c58a19ddb,37267,1731751077844 2024-11-16T10:00:34,718 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731751217321.75a5a39ee7bf008864556f27870615b9., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=75a5a39ee7bf008864556f27870615b9, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:00:34,718 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731751217321.75a5a39ee7bf008864556f27870615b9., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 because the exception is null or not the one we care about 2024-11-16T10:00:34,718 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731751217321.75a5a39ee7bf008864556f27870615b9., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 , the old value is region=TestAcidGuarantees,,1731751217321.75a5a39ee7bf008864556f27870615b9., hostname=037c58a19ddb,37267,1731751077844, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=75a5a39ee7bf008864556f27870615b9, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:00:34,718 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731751217321.75a5a39ee7bf008864556f27870615b9., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=75a5a39ee7bf008864556f27870615b9, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:00:34,718 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731751217321.75a5a39ee7bf008864556f27870615b9., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 because the exception is null or not the one we care about 2024-11-16T10:00:34,718 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=75a5a39ee7bf008864556f27870615b9, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T10:00:34,719 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] ipc.CallRunner(138): callId: 81 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34392 deadline: 1731751294715, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=75a5a39ee7bf008864556f27870615b9, server=037c58a19ddb,37267,1731751077844 2024-11-16T10:00:34,719 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731751217321.75a5a39ee7bf008864556f27870615b9., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 , the old value is region=TestAcidGuarantees,,1731751217321.75a5a39ee7bf008864556f27870615b9., hostname=037c58a19ddb,37267,1731751077844, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=75a5a39ee7bf008864556f27870615b9, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:00:34,719 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731751217321.75a5a39ee7bf008864556f27870615b9., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=75a5a39ee7bf008864556f27870615b9, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:00:34,719 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731751217321.75a5a39ee7bf008864556f27870615b9., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 because the exception is null or not the one we care about 2024-11-16T10:00:34,720 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=75a5a39ee7bf008864556f27870615b9, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T10:00:34,720 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] ipc.CallRunner(138): callId: 77 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34428 deadline: 1731751294718, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=75a5a39ee7bf008864556f27870615b9, server=037c58a19ddb,37267,1731751077844 2024-11-16T10:00:34,721 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731751217321.75a5a39ee7bf008864556f27870615b9., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 , the old value is region=TestAcidGuarantees,,1731751217321.75a5a39ee7bf008864556f27870615b9., hostname=037c58a19ddb,37267,1731751077844, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=75a5a39ee7bf008864556f27870615b9, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:00:34,721 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731751217321.75a5a39ee7bf008864556f27870615b9., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=75a5a39ee7bf008864556f27870615b9, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:00:34,721 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731751217321.75a5a39ee7bf008864556f27870615b9., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 because the exception is null or not the one we care about 2024-11-16T10:00:34,722 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=75a5a39ee7bf008864556f27870615b9, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T10:00:34,723 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] ipc.CallRunner(138): callId: 89 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34412 deadline: 1731751294719, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=75a5a39ee7bf008864556f27870615b9, server=037c58a19ddb,37267,1731751077844 2024-11-16T10:00:34,723 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731751217321.75a5a39ee7bf008864556f27870615b9., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 , the old value is region=TestAcidGuarantees,,1731751217321.75a5a39ee7bf008864556f27870615b9., hostname=037c58a19ddb,37267,1731751077844, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=75a5a39ee7bf008864556f27870615b9, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:00:34,723 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731751217321.75a5a39ee7bf008864556f27870615b9., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=75a5a39ee7bf008864556f27870615b9, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:00:34,723 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731751217321.75a5a39ee7bf008864556f27870615b9., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 because the exception is null or not the one we care about 2024-11-16T10:00:34,825 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=75a5a39ee7bf008864556f27870615b9, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T10:00:34,825 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] ipc.CallRunner(138): callId: 85 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34430 deadline: 1731751294822, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=75a5a39ee7bf008864556f27870615b9, server=037c58a19ddb,37267,1731751077844 2024-11-16T10:00:34,825 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=75a5a39ee7bf008864556f27870615b9, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T10:00:34,825 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] ipc.CallRunner(138): callId: 82 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34454 deadline: 1731751294822, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=75a5a39ee7bf008864556f27870615b9, server=037c58a19ddb,37267,1731751077844 2024-11-16T10:00:34,825 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731751217321.75a5a39ee7bf008864556f27870615b9., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 , the old value is region=TestAcidGuarantees,,1731751217321.75a5a39ee7bf008864556f27870615b9., hostname=037c58a19ddb,37267,1731751077844, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=75a5a39ee7bf008864556f27870615b9, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:00:34,825 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731751217321.75a5a39ee7bf008864556f27870615b9., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=75a5a39ee7bf008864556f27870615b9, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:00:34,825 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731751217321.75a5a39ee7bf008864556f27870615b9., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 because the exception is null or not the one we care about 2024-11-16T10:00:34,825 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=75a5a39ee7bf008864556f27870615b9, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T10:00:34,825 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731751217321.75a5a39ee7bf008864556f27870615b9., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 , the old value is region=TestAcidGuarantees,,1731751217321.75a5a39ee7bf008864556f27870615b9., hostname=037c58a19ddb,37267,1731751077844, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=75a5a39ee7bf008864556f27870615b9, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:00:34,825 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] ipc.CallRunner(138): callId: 82 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34392 deadline: 1731751294823, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=75a5a39ee7bf008864556f27870615b9, server=037c58a19ddb,37267,1731751077844 2024-11-16T10:00:34,825 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731751217321.75a5a39ee7bf008864556f27870615b9., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=75a5a39ee7bf008864556f27870615b9, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:00:34,825 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731751217321.75a5a39ee7bf008864556f27870615b9., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 because the exception is null or not the one we care about 2024-11-16T10:00:34,826 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=75a5a39ee7bf008864556f27870615b9, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T10:00:34,826 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731751217321.75a5a39ee7bf008864556f27870615b9., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 , the old value is region=TestAcidGuarantees,,1731751217321.75a5a39ee7bf008864556f27870615b9., hostname=037c58a19ddb,37267,1731751077844, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=75a5a39ee7bf008864556f27870615b9, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:00:34,826 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731751217321.75a5a39ee7bf008864556f27870615b9., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=75a5a39ee7bf008864556f27870615b9, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:00:34,826 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] ipc.CallRunner(138): callId: 78 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34428 deadline: 1731751294823, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=75a5a39ee7bf008864556f27870615b9, server=037c58a19ddb,37267,1731751077844 2024-11-16T10:00:34,826 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731751217321.75a5a39ee7bf008864556f27870615b9., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 because the exception is null or not the one we care about 2024-11-16T10:00:34,826 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731751217321.75a5a39ee7bf008864556f27870615b9., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 , the old value is region=TestAcidGuarantees,,1731751217321.75a5a39ee7bf008864556f27870615b9., hostname=037c58a19ddb,37267,1731751077844, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=75a5a39ee7bf008864556f27870615b9, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:00:34,826 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731751217321.75a5a39ee7bf008864556f27870615b9., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=75a5a39ee7bf008864556f27870615b9, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:00:34,826 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731751217321.75a5a39ee7bf008864556f27870615b9., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 because the exception is null or not the one we care about 2024-11-16T10:00:34,835 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=75a5a39ee7bf008864556f27870615b9, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T10:00:34,835 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] ipc.CallRunner(138): callId: 90 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34412 deadline: 1731751294833, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=75a5a39ee7bf008864556f27870615b9, server=037c58a19ddb,37267,1731751077844 2024-11-16T10:00:34,836 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731751217321.75a5a39ee7bf008864556f27870615b9., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 , the old value is region=TestAcidGuarantees,,1731751217321.75a5a39ee7bf008864556f27870615b9., hostname=037c58a19ddb,37267,1731751077844, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=75a5a39ee7bf008864556f27870615b9, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:00:34,836 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731751217321.75a5a39ee7bf008864556f27870615b9., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=75a5a39ee7bf008864556f27870615b9, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:00:34,836 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731751217321.75a5a39ee7bf008864556f27870615b9., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 because the exception is null or not the one we care about 2024-11-16T10:00:35,035 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=75a5a39ee7bf008864556f27870615b9, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T10:00:35,035 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] ipc.CallRunner(138): callId: 86 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34430 deadline: 1731751295032, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=75a5a39ee7bf008864556f27870615b9, server=037c58a19ddb,37267,1731751077844 2024-11-16T10:00:35,035 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=75a5a39ee7bf008864556f27870615b9, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T10:00:35,035 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] ipc.CallRunner(138): callId: 83 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34454 deadline: 1731751295033, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=75a5a39ee7bf008864556f27870615b9, server=037c58a19ddb,37267,1731751077844 2024-11-16T10:00:35,035 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=75a5a39ee7bf008864556f27870615b9, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T10:00:35,035 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731751217321.75a5a39ee7bf008864556f27870615b9., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 , the old value is region=TestAcidGuarantees,,1731751217321.75a5a39ee7bf008864556f27870615b9., hostname=037c58a19ddb,37267,1731751077844, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=75a5a39ee7bf008864556f27870615b9, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:00:35,035 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] ipc.CallRunner(138): callId: 83 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34392 deadline: 1731751295033, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=75a5a39ee7bf008864556f27870615b9, server=037c58a19ddb,37267,1731751077844 2024-11-16T10:00:35,035 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731751217321.75a5a39ee7bf008864556f27870615b9., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=75a5a39ee7bf008864556f27870615b9, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:00:35,035 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731751217321.75a5a39ee7bf008864556f27870615b9., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 because the exception is null or not the one we care about 2024-11-16T10:00:35,035 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=75a5a39ee7bf008864556f27870615b9, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T10:00:35,035 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731751217321.75a5a39ee7bf008864556f27870615b9., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 , the old value is region=TestAcidGuarantees,,1731751217321.75a5a39ee7bf008864556f27870615b9., hostname=037c58a19ddb,37267,1731751077844, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=75a5a39ee7bf008864556f27870615b9, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:00:35,035 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731751217321.75a5a39ee7bf008864556f27870615b9., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=75a5a39ee7bf008864556f27870615b9, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:00:35,035 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] ipc.CallRunner(138): callId: 79 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34428 deadline: 1731751295033, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=75a5a39ee7bf008864556f27870615b9, server=037c58a19ddb,37267,1731751077844 2024-11-16T10:00:35,035 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731751217321.75a5a39ee7bf008864556f27870615b9., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 because the exception is null or not the one we care about 2024-11-16T10:00:35,035 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731751217321.75a5a39ee7bf008864556f27870615b9., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 , the old value is region=TestAcidGuarantees,,1731751217321.75a5a39ee7bf008864556f27870615b9., hostname=037c58a19ddb,37267,1731751077844, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=75a5a39ee7bf008864556f27870615b9, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:00:35,036 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731751217321.75a5a39ee7bf008864556f27870615b9., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=75a5a39ee7bf008864556f27870615b9, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:00:35,036 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731751217321.75a5a39ee7bf008864556f27870615b9., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 because the exception is null or not the one we care about 2024-11-16T10:00:35,036 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731751217321.75a5a39ee7bf008864556f27870615b9., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 , the old value is region=TestAcidGuarantees,,1731751217321.75a5a39ee7bf008864556f27870615b9., hostname=037c58a19ddb,37267,1731751077844, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=75a5a39ee7bf008864556f27870615b9, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:00:35,036 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731751217321.75a5a39ee7bf008864556f27870615b9., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=75a5a39ee7bf008864556f27870615b9, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:00:35,036 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731751217321.75a5a39ee7bf008864556f27870615b9., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 because the exception is null or not the one we care about 2024-11-16T10:00:35,045 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=75a5a39ee7bf008864556f27870615b9, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T10:00:35,045 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] ipc.CallRunner(138): callId: 91 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34412 deadline: 1731751295043, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=75a5a39ee7bf008864556f27870615b9, server=037c58a19ddb,37267,1731751077844 2024-11-16T10:00:35,045 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731751217321.75a5a39ee7bf008864556f27870615b9., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 , the old value is region=TestAcidGuarantees,,1731751217321.75a5a39ee7bf008864556f27870615b9., hostname=037c58a19ddb,37267,1731751077844, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=75a5a39ee7bf008864556f27870615b9, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:00:35,045 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731751217321.75a5a39ee7bf008864556f27870615b9., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=75a5a39ee7bf008864556f27870615b9, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:00:35,045 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731751217321.75a5a39ee7bf008864556f27870615b9., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 because the exception is null or not the one we care about 2024-11-16T10:00:35,089 INFO [RS_FLUSH_OPERATIONS-regionserver/037c58a19ddb:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=246 (bloomFilter=true), to=hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/75a5a39ee7bf008864556f27870615b9/.tmp/C/f158f302c9e4447abd35ad6d46fa5b3e 2024-11-16T10:00:35,092 DEBUG [RS_FLUSH_OPERATIONS-regionserver/037c58a19ddb:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/75a5a39ee7bf008864556f27870615b9/.tmp/A/a15a55fc49b74b35997765299aa047dc as hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/75a5a39ee7bf008864556f27870615b9/A/a15a55fc49b74b35997765299aa047dc 2024-11-16T10:00:35,095 INFO [RS_FLUSH_OPERATIONS-regionserver/037c58a19ddb:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/75a5a39ee7bf008864556f27870615b9/A/a15a55fc49b74b35997765299aa047dc, entries=150, sequenceid=246, filesize=30.4 K 2024-11-16T10:00:35,097 DEBUG [RS_FLUSH_OPERATIONS-regionserver/037c58a19ddb:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/75a5a39ee7bf008864556f27870615b9/.tmp/B/e59fe4cb81fc403999f5bb2b146958fc as hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/75a5a39ee7bf008864556f27870615b9/B/e59fe4cb81fc403999f5bb2b146958fc 2024-11-16T10:00:35,100 INFO [RS_FLUSH_OPERATIONS-regionserver/037c58a19ddb:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/75a5a39ee7bf008864556f27870615b9/B/e59fe4cb81fc403999f5bb2b146958fc, entries=150, sequenceid=246, filesize=11.9 K 2024-11-16T10:00:35,101 DEBUG [RS_FLUSH_OPERATIONS-regionserver/037c58a19ddb:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/75a5a39ee7bf008864556f27870615b9/.tmp/C/f158f302c9e4447abd35ad6d46fa5b3e as hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/75a5a39ee7bf008864556f27870615b9/C/f158f302c9e4447abd35ad6d46fa5b3e 2024-11-16T10:00:35,105 INFO [RS_FLUSH_OPERATIONS-regionserver/037c58a19ddb:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/75a5a39ee7bf008864556f27870615b9/C/f158f302c9e4447abd35ad6d46fa5b3e, entries=150, sequenceid=246, filesize=11.9 K 2024-11-16T10:00:35,105 INFO [RS_FLUSH_OPERATIONS-regionserver/037c58a19ddb:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.HRegion(3140): Finished flush of dataSize ~33.54 KB/34350, heapSize ~88.59 KB/90720, currentSize=174.43 KB/178620 for 75a5a39ee7bf008864556f27870615b9 in 895ms, sequenceid=246, compaction requested=false 2024-11-16T10:00:35,106 DEBUG [RS_FLUSH_OPERATIONS-regionserver/037c58a19ddb:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.HRegion(2603): Flush status journal for 75a5a39ee7bf008864556f27870615b9: 2024-11-16T10:00:35,106 DEBUG [RS_FLUSH_OPERATIONS-regionserver/037c58a19ddb:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731751217321.75a5a39ee7bf008864556f27870615b9. 2024-11-16T10:00:35,106 DEBUG [RS_FLUSH_OPERATIONS-regionserver/037c58a19ddb:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=161 2024-11-16T10:00:35,106 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37877 {}] master.HMaster(4169): Remote procedure done, pid=161 2024-11-16T10:00:35,109 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=161, resume processing ppid=160 2024-11-16T10:00:35,109 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=161, ppid=160, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.7300 sec 2024-11-16T10:00:35,111 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=160, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=160, table=TestAcidGuarantees in 2.7360 sec 2024-11-16T10:00:35,346 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] regionserver.HRegion(8855): Flush requested on 75a5a39ee7bf008864556f27870615b9 2024-11-16T10:00:35,346 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-16T10:00:35,346 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 75a5a39ee7bf008864556f27870615b9 3/3 column families, dataSize=181.14 KB heapSize=475.36 KB 2024-11-16T10:00:35,350 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=75a5a39ee7bf008864556f27870615b9, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T10:00:35,350 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] ipc.CallRunner(138): callId: 88 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34430 deadline: 1731751295346, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=75a5a39ee7bf008864556f27870615b9, server=037c58a19ddb,37267,1731751077844 2024-11-16T10:00:35,350 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=75a5a39ee7bf008864556f27870615b9, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T10:00:35,350 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] ipc.CallRunner(138): callId: 85 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34454 deadline: 1731751295347, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=75a5a39ee7bf008864556f27870615b9, server=037c58a19ddb,37267,1731751077844 2024-11-16T10:00:35,350 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731751217321.75a5a39ee7bf008864556f27870615b9., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 , the old value is region=TestAcidGuarantees,,1731751217321.75a5a39ee7bf008864556f27870615b9., hostname=037c58a19ddb,37267,1731751077844, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=75a5a39ee7bf008864556f27870615b9, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:00:35,350 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731751217321.75a5a39ee7bf008864556f27870615b9., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=75a5a39ee7bf008864556f27870615b9, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:00:35,350 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731751217321.75a5a39ee7bf008864556f27870615b9., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 because the exception is null or not the one we care about 2024-11-16T10:00:35,351 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731751217321.75a5a39ee7bf008864556f27870615b9., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 , the old value is region=TestAcidGuarantees,,1731751217321.75a5a39ee7bf008864556f27870615b9., hostname=037c58a19ddb,37267,1731751077844, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=75a5a39ee7bf008864556f27870615b9, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:00:35,351 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731751217321.75a5a39ee7bf008864556f27870615b9., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=75a5a39ee7bf008864556f27870615b9, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:00:35,351 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731751217321.75a5a39ee7bf008864556f27870615b9., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 because the exception is null or not the one we care about 2024-11-16T10:00:35,351 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=75a5a39ee7bf008864556f27870615b9, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T10:00:35,351 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] ipc.CallRunner(138): callId: 85 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34392 deadline: 1731751295347, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=75a5a39ee7bf008864556f27870615b9, server=037c58a19ddb,37267,1731751077844 2024-11-16T10:00:35,351 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=75a5a39ee7bf008864556f27870615b9, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T10:00:35,351 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] ipc.CallRunner(138): callId: 81 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34428 deadline: 1731751295349, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=75a5a39ee7bf008864556f27870615b9, server=037c58a19ddb,37267,1731751077844 2024-11-16T10:00:35,351 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731751217321.75a5a39ee7bf008864556f27870615b9., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 , the old value is region=TestAcidGuarantees,,1731751217321.75a5a39ee7bf008864556f27870615b9., hostname=037c58a19ddb,37267,1731751077844, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=75a5a39ee7bf008864556f27870615b9, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:00:35,351 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731751217321.75a5a39ee7bf008864556f27870615b9., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=75a5a39ee7bf008864556f27870615b9, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:00:35,351 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731751217321.75a5a39ee7bf008864556f27870615b9., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 because the exception is null or not the one we care about 2024-11-16T10:00:35,351 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731751217321.75a5a39ee7bf008864556f27870615b9., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 , the old value is region=TestAcidGuarantees,,1731751217321.75a5a39ee7bf008864556f27870615b9., hostname=037c58a19ddb,37267,1731751077844, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=75a5a39ee7bf008864556f27870615b9, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:00:35,351 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731751217321.75a5a39ee7bf008864556f27870615b9., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=75a5a39ee7bf008864556f27870615b9, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:00:35,351 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731751217321.75a5a39ee7bf008864556f27870615b9., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 because the exception is null or not the one we care about 2024-11-16T10:00:35,355 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=75a5a39ee7bf008864556f27870615b9, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T10:00:35,355 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] ipc.CallRunner(138): callId: 92 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34412 deadline: 1731751295353, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=75a5a39ee7bf008864556f27870615b9, server=037c58a19ddb,37267,1731751077844 2024-11-16T10:00:35,355 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731751217321.75a5a39ee7bf008864556f27870615b9., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 , the old value is region=TestAcidGuarantees,,1731751217321.75a5a39ee7bf008864556f27870615b9., hostname=037c58a19ddb,37267,1731751077844, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=75a5a39ee7bf008864556f27870615b9, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:00:35,355 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731751217321.75a5a39ee7bf008864556f27870615b9., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=75a5a39ee7bf008864556f27870615b9, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:00:35,355 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731751217321.75a5a39ee7bf008864556f27870615b9., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 because the exception is null or not the one we care about 2024-11-16T10:00:35,355 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241116e29db2783f8f45f1bf1c01e0b8271a4e_75a5a39ee7bf008864556f27870615b9 is 50, key is test_row_0/A:col10/1731751235346/Put/seqid=0 2024-11-16T10:00:35,359 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37591 is added to blk_1073742439_1615 (size=14994) 2024-11-16T10:00:35,359 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:35,362 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241116e29db2783f8f45f1bf1c01e0b8271a4e_75a5a39ee7bf008864556f27870615b9 to hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241116e29db2783f8f45f1bf1c01e0b8271a4e_75a5a39ee7bf008864556f27870615b9 2024-11-16T10:00:35,372 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/75a5a39ee7bf008864556f27870615b9/.tmp/A/889382ab4777414f88c6fe4df51e78eb, store: [table=TestAcidGuarantees family=A region=75a5a39ee7bf008864556f27870615b9] 2024-11-16T10:00:35,373 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/75a5a39ee7bf008864556f27870615b9/.tmp/A/889382ab4777414f88c6fe4df51e78eb is 175, key is test_row_0/A:col10/1731751235346/Put/seqid=0 2024-11-16T10:00:35,386 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37591 is added to blk_1073742440_1616 (size=39949) 2024-11-16T10:00:35,454 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=75a5a39ee7bf008864556f27870615b9, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T10:00:35,454 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] ipc.CallRunner(138): callId: 89 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34430 deadline: 1731751295452, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=75a5a39ee7bf008864556f27870615b9, server=037c58a19ddb,37267,1731751077844 2024-11-16T10:00:35,454 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=75a5a39ee7bf008864556f27870615b9, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T10:00:35,454 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731751217321.75a5a39ee7bf008864556f27870615b9., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 , the old value is region=TestAcidGuarantees,,1731751217321.75a5a39ee7bf008864556f27870615b9., hostname=037c58a19ddb,37267,1731751077844, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=75a5a39ee7bf008864556f27870615b9, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:00:35,454 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731751217321.75a5a39ee7bf008864556f27870615b9., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=75a5a39ee7bf008864556f27870615b9, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:00:35,454 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731751217321.75a5a39ee7bf008864556f27870615b9., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 because the exception is null or not the one we care about 2024-11-16T10:00:35,454 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] ipc.CallRunner(138): callId: 86 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34454 deadline: 1731751295453, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=75a5a39ee7bf008864556f27870615b9, server=037c58a19ddb,37267,1731751077844 2024-11-16T10:00:35,454 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=75a5a39ee7bf008864556f27870615b9, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T10:00:35,455 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731751217321.75a5a39ee7bf008864556f27870615b9., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 , the old value is region=TestAcidGuarantees,,1731751217321.75a5a39ee7bf008864556f27870615b9., hostname=037c58a19ddb,37267,1731751077844, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=75a5a39ee7bf008864556f27870615b9, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:00:35,455 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731751217321.75a5a39ee7bf008864556f27870615b9., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=75a5a39ee7bf008864556f27870615b9, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:00:35,455 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] ipc.CallRunner(138): callId: 86 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34392 deadline: 1731751295453, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=75a5a39ee7bf008864556f27870615b9, server=037c58a19ddb,37267,1731751077844 2024-11-16T10:00:35,455 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731751217321.75a5a39ee7bf008864556f27870615b9., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 because the exception is null or not the one we care about 2024-11-16T10:00:35,455 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731751217321.75a5a39ee7bf008864556f27870615b9., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 , the old value is region=TestAcidGuarantees,,1731751217321.75a5a39ee7bf008864556f27870615b9., hostname=037c58a19ddb,37267,1731751077844, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=75a5a39ee7bf008864556f27870615b9, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:00:35,455 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731751217321.75a5a39ee7bf008864556f27870615b9., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=75a5a39ee7bf008864556f27870615b9, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:00:35,455 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731751217321.75a5a39ee7bf008864556f27870615b9., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 because the exception is null or not the one we care about 2024-11-16T10:00:35,464 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=75a5a39ee7bf008864556f27870615b9, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T10:00:35,464 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] ipc.CallRunner(138): callId: 82 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34428 deadline: 1731751295463, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=75a5a39ee7bf008864556f27870615b9, server=037c58a19ddb,37267,1731751077844 2024-11-16T10:00:35,465 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731751217321.75a5a39ee7bf008864556f27870615b9., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 , the old value is region=TestAcidGuarantees,,1731751217321.75a5a39ee7bf008864556f27870615b9., hostname=037c58a19ddb,37267,1731751077844, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=75a5a39ee7bf008864556f27870615b9, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:00:35,465 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731751217321.75a5a39ee7bf008864556f27870615b9., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=75a5a39ee7bf008864556f27870615b9, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:00:35,465 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731751217321.75a5a39ee7bf008864556f27870615b9., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 because the exception is null or not the one we care about 2024-11-16T10:00:35,664 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=75a5a39ee7bf008864556f27870615b9, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T10:00:35,664 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] ipc.CallRunner(138): callId: 90 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34430 deadline: 1731751295663, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=75a5a39ee7bf008864556f27870615b9, server=037c58a19ddb,37267,1731751077844 2024-11-16T10:00:35,664 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731751217321.75a5a39ee7bf008864556f27870615b9., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 , the old value is region=TestAcidGuarantees,,1731751217321.75a5a39ee7bf008864556f27870615b9., hostname=037c58a19ddb,37267,1731751077844, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=75a5a39ee7bf008864556f27870615b9, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:00:35,664 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731751217321.75a5a39ee7bf008864556f27870615b9., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=75a5a39ee7bf008864556f27870615b9, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:00:35,664 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731751217321.75a5a39ee7bf008864556f27870615b9., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 because the exception is null or not the one we care about 2024-11-16T10:00:35,667 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=75a5a39ee7bf008864556f27870615b9, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T10:00:35,667 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] ipc.CallRunner(138): callId: 87 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34392 deadline: 1731751295666, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=75a5a39ee7bf008864556f27870615b9, server=037c58a19ddb,37267,1731751077844 2024-11-16T10:00:35,667 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=75a5a39ee7bf008864556f27870615b9, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T10:00:35,668 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] ipc.CallRunner(138): callId: 87 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34454 deadline: 1731751295666, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=75a5a39ee7bf008864556f27870615b9, server=037c58a19ddb,37267,1731751077844 2024-11-16T10:00:35,668 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731751217321.75a5a39ee7bf008864556f27870615b9., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 , the old value is region=TestAcidGuarantees,,1731751217321.75a5a39ee7bf008864556f27870615b9., hostname=037c58a19ddb,37267,1731751077844, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=75a5a39ee7bf008864556f27870615b9, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:00:35,668 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731751217321.75a5a39ee7bf008864556f27870615b9., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=75a5a39ee7bf008864556f27870615b9, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:00:35,668 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731751217321.75a5a39ee7bf008864556f27870615b9., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 because the exception is null or not the one we care about 2024-11-16T10:00:35,668 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731751217321.75a5a39ee7bf008864556f27870615b9., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 , the old value is region=TestAcidGuarantees,,1731751217321.75a5a39ee7bf008864556f27870615b9., hostname=037c58a19ddb,37267,1731751077844, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=75a5a39ee7bf008864556f27870615b9, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:00:35,668 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731751217321.75a5a39ee7bf008864556f27870615b9., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=75a5a39ee7bf008864556f27870615b9, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:00:35,668 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731751217321.75a5a39ee7bf008864556f27870615b9., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 because the exception is null or not the one we care about 2024-11-16T10:00:35,674 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=75a5a39ee7bf008864556f27870615b9, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T10:00:35,674 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] ipc.CallRunner(138): callId: 83 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34428 deadline: 1731751295673, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=75a5a39ee7bf008864556f27870615b9, server=037c58a19ddb,37267,1731751077844 2024-11-16T10:00:35,675 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731751217321.75a5a39ee7bf008864556f27870615b9., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 , the old value is region=TestAcidGuarantees,,1731751217321.75a5a39ee7bf008864556f27870615b9., hostname=037c58a19ddb,37267,1731751077844, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=75a5a39ee7bf008864556f27870615b9, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:00:35,675 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731751217321.75a5a39ee7bf008864556f27870615b9., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=75a5a39ee7bf008864556f27870615b9, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:00:35,675 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731751217321.75a5a39ee7bf008864556f27870615b9., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 because the exception is null or not the one we care about 2024-11-16T10:00:35,787 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=281, memsize=64.9 K, hasBloomFilter=true, into tmp file hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/75a5a39ee7bf008864556f27870615b9/.tmp/A/889382ab4777414f88c6fe4df51e78eb 2024-11-16T10:00:35,793 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/75a5a39ee7bf008864556f27870615b9/.tmp/B/4c0a41e3bba648e79161d541f37c4939 is 50, key is test_row_0/B:col10/1731751235346/Put/seqid=0 2024-11-16T10:00:35,796 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37591 is added to blk_1073742441_1617 (size=12301) 2024-11-16T10:00:35,864 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=75a5a39ee7bf008864556f27870615b9, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T10:00:35,865 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] ipc.CallRunner(138): callId: 93 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34412 deadline: 1731751295862, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=75a5a39ee7bf008864556f27870615b9, server=037c58a19ddb,37267,1731751077844 2024-11-16T10:00:35,865 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731751217321.75a5a39ee7bf008864556f27870615b9., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 , the old value is region=TestAcidGuarantees,,1731751217321.75a5a39ee7bf008864556f27870615b9., hostname=037c58a19ddb,37267,1731751077844, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=75a5a39ee7bf008864556f27870615b9, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:00:35,865 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731751217321.75a5a39ee7bf008864556f27870615b9., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=75a5a39ee7bf008864556f27870615b9, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:00:35,865 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731751217321.75a5a39ee7bf008864556f27870615b9., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 because the exception is null or not the one we care about 2024-11-16T10:00:35,974 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=75a5a39ee7bf008864556f27870615b9, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T10:00:35,974 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] ipc.CallRunner(138): callId: 91 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34430 deadline: 1731751295972, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=75a5a39ee7bf008864556f27870615b9, server=037c58a19ddb,37267,1731751077844 2024-11-16T10:00:35,974 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731751217321.75a5a39ee7bf008864556f27870615b9., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 , the old value is region=TestAcidGuarantees,,1731751217321.75a5a39ee7bf008864556f27870615b9., hostname=037c58a19ddb,37267,1731751077844, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=75a5a39ee7bf008864556f27870615b9, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:00:35,974 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=75a5a39ee7bf008864556f27870615b9, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T10:00:35,974 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=75a5a39ee7bf008864556f27870615b9, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T10:00:35,974 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731751217321.75a5a39ee7bf008864556f27870615b9., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=75a5a39ee7bf008864556f27870615b9, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:00:35,974 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731751217321.75a5a39ee7bf008864556f27870615b9., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 because the exception is null or not the one we care about 2024-11-16T10:00:35,974 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] ipc.CallRunner(138): callId: 88 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34454 deadline: 1731751295973, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=75a5a39ee7bf008864556f27870615b9, server=037c58a19ddb,37267,1731751077844 2024-11-16T10:00:35,974 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] ipc.CallRunner(138): callId: 88 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34392 deadline: 1731751295972, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=75a5a39ee7bf008864556f27870615b9, server=037c58a19ddb,37267,1731751077844 2024-11-16T10:00:35,974 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731751217321.75a5a39ee7bf008864556f27870615b9., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 , the old value is region=TestAcidGuarantees,,1731751217321.75a5a39ee7bf008864556f27870615b9., hostname=037c58a19ddb,37267,1731751077844, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=75a5a39ee7bf008864556f27870615b9, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:00:35,974 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731751217321.75a5a39ee7bf008864556f27870615b9., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=75a5a39ee7bf008864556f27870615b9, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:00:35,974 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731751217321.75a5a39ee7bf008864556f27870615b9., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 because the exception is null or not the one we care about 2024-11-16T10:00:35,974 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731751217321.75a5a39ee7bf008864556f27870615b9., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 , the old value is region=TestAcidGuarantees,,1731751217321.75a5a39ee7bf008864556f27870615b9., hostname=037c58a19ddb,37267,1731751077844, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=75a5a39ee7bf008864556f27870615b9, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:00:35,975 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731751217321.75a5a39ee7bf008864556f27870615b9., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=75a5a39ee7bf008864556f27870615b9, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:00:35,975 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731751217321.75a5a39ee7bf008864556f27870615b9., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 because the exception is null or not the one we care about 2024-11-16T10:00:35,984 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=75a5a39ee7bf008864556f27870615b9, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T10:00:35,984 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] ipc.CallRunner(138): callId: 84 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34428 deadline: 1731751295983, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=75a5a39ee7bf008864556f27870615b9, server=037c58a19ddb,37267,1731751077844 2024-11-16T10:00:35,984 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731751217321.75a5a39ee7bf008864556f27870615b9., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 , the old value is region=TestAcidGuarantees,,1731751217321.75a5a39ee7bf008864556f27870615b9., hostname=037c58a19ddb,37267,1731751077844, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=75a5a39ee7bf008864556f27870615b9, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:00:35,984 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731751217321.75a5a39ee7bf008864556f27870615b9., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=75a5a39ee7bf008864556f27870615b9, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:00:35,984 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731751217321.75a5a39ee7bf008864556f27870615b9., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 because the exception is null or not the one we care about 2024-11-16T10:00:36,197 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=64.85 KB at sequenceid=281 (bloomFilter=true), to=hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/75a5a39ee7bf008864556f27870615b9/.tmp/B/4c0a41e3bba648e79161d541f37c4939 2024-11-16T10:00:36,210 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/75a5a39ee7bf008864556f27870615b9/.tmp/C/2f41815c61aa4891bf17f02eb08cfa8e is 50, key is test_row_0/C:col10/1731751235346/Put/seqid=0 2024-11-16T10:00:36,214 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37591 is added to blk_1073742442_1618 (size=12301) 2024-11-16T10:00:36,214 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=64.85 KB at sequenceid=281 (bloomFilter=true), to=hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/75a5a39ee7bf008864556f27870615b9/.tmp/C/2f41815c61aa4891bf17f02eb08cfa8e 2024-11-16T10:00:36,226 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/75a5a39ee7bf008864556f27870615b9/.tmp/A/889382ab4777414f88c6fe4df51e78eb as hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/75a5a39ee7bf008864556f27870615b9/A/889382ab4777414f88c6fe4df51e78eb 2024-11-16T10:00:36,229 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/75a5a39ee7bf008864556f27870615b9/A/889382ab4777414f88c6fe4df51e78eb, entries=200, sequenceid=281, filesize=39.0 K 2024-11-16T10:00:36,230 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/75a5a39ee7bf008864556f27870615b9/.tmp/B/4c0a41e3bba648e79161d541f37c4939 as hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/75a5a39ee7bf008864556f27870615b9/B/4c0a41e3bba648e79161d541f37c4939 2024-11-16T10:00:36,231 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:36,231 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:36,233 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:36,233 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/75a5a39ee7bf008864556f27870615b9/B/4c0a41e3bba648e79161d541f37c4939, entries=150, sequenceid=281, filesize=12.0 K 2024-11-16T10:00:36,234 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/75a5a39ee7bf008864556f27870615b9/.tmp/C/2f41815c61aa4891bf17f02eb08cfa8e as hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/75a5a39ee7bf008864556f27870615b9/C/2f41815c61aa4891bf17f02eb08cfa8e 2024-11-16T10:00:36,234 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:36,234 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:36,234 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:36,235 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:36,235 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:36,235 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:36,235 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:36,236 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:36,236 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:36,236 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:36,236 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:36,236 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:36,237 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:36,237 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:36,237 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:36,237 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:36,237 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:36,237 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:36,238 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/75a5a39ee7bf008864556f27870615b9/C/2f41815c61aa4891bf17f02eb08cfa8e, entries=150, sequenceid=281, filesize=12.0 K 2024-11-16T10:00:36,238 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:36,238 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:36,238 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:36,239 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:36,239 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:36,239 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:36,239 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~194.56 KB/199230, heapSize ~510.47 KB/522720, currentSize=6.71 KB/6870 for 75a5a39ee7bf008864556f27870615b9 in 893ms, sequenceid=281, compaction requested=true 2024-11-16T10:00:36,239 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 75a5a39ee7bf008864556f27870615b9: 2024-11-16T10:00:36,239 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 75a5a39ee7bf008864556f27870615b9:A, priority=-2147483648, current under compaction store size is 1 2024-11-16T10:00:36,239 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-16T10:00:36,239 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:36,239 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:36,239 DEBUG [RS:0;037c58a19ddb:37267-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-16T10:00:36,239 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 75a5a39ee7bf008864556f27870615b9:B, priority=-2147483648, current under compaction store size is 2 2024-11-16T10:00:36,239 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-16T10:00:36,239 DEBUG [RS:0;037c58a19ddb:37267-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-16T10:00:36,239 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 75a5a39ee7bf008864556f27870615b9:C, priority=-2147483648, current under compaction store size is 3 2024-11-16T10:00:36,239 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-16T10:00:36,240 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:36,240 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:36,240 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:36,240 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:36,240 DEBUG [RS:0;037c58a19ddb:37267-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37149 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-16T10:00:36,240 DEBUG [RS:0;037c58a19ddb:37267-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 102705 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-16T10:00:36,241 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:36,241 DEBUG [RS:0;037c58a19ddb:37267-shortCompactions-0 {}] regionserver.HStore(1541): 75a5a39ee7bf008864556f27870615b9/A is initiating minor compaction (all files) 2024-11-16T10:00:36,241 DEBUG [RS:0;037c58a19ddb:37267-longCompactions-0 {}] regionserver.HStore(1541): 75a5a39ee7bf008864556f27870615b9/B is initiating minor compaction (all files) 2024-11-16T10:00:36,241 INFO [RS:0;037c58a19ddb:37267-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 75a5a39ee7bf008864556f27870615b9/A in TestAcidGuarantees,,1731751217321.75a5a39ee7bf008864556f27870615b9. 2024-11-16T10:00:36,241 INFO [RS:0;037c58a19ddb:37267-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 75a5a39ee7bf008864556f27870615b9/B in TestAcidGuarantees,,1731751217321.75a5a39ee7bf008864556f27870615b9. 2024-11-16T10:00:36,241 INFO [RS:0;037c58a19ddb:37267-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/75a5a39ee7bf008864556f27870615b9/B/2dac6b2aba484c8ca3588f74e5a36789, hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/75a5a39ee7bf008864556f27870615b9/B/e59fe4cb81fc403999f5bb2b146958fc, hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/75a5a39ee7bf008864556f27870615b9/B/4c0a41e3bba648e79161d541f37c4939] into tmpdir=hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/75a5a39ee7bf008864556f27870615b9/.tmp, totalSize=36.3 K 2024-11-16T10:00:36,241 INFO [RS:0;037c58a19ddb:37267-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/75a5a39ee7bf008864556f27870615b9/A/f234c94f31c345bdacdbdad119738a8c, hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/75a5a39ee7bf008864556f27870615b9/A/a15a55fc49b74b35997765299aa047dc, hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/75a5a39ee7bf008864556f27870615b9/A/889382ab4777414f88c6fe4df51e78eb] into tmpdir=hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/75a5a39ee7bf008864556f27870615b9/.tmp, totalSize=100.3 K 2024-11-16T10:00:36,241 INFO [RS:0;037c58a19ddb:37267-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731751217321.75a5a39ee7bf008864556f27870615b9. 2024-11-16T10:00:36,241 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:36,241 DEBUG [RS:0;037c58a19ddb:37267-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731751217321.75a5a39ee7bf008864556f27870615b9. files: [hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/75a5a39ee7bf008864556f27870615b9/A/f234c94f31c345bdacdbdad119738a8c, hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/75a5a39ee7bf008864556f27870615b9/A/a15a55fc49b74b35997765299aa047dc, hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/75a5a39ee7bf008864556f27870615b9/A/889382ab4777414f88c6fe4df51e78eb] 2024-11-16T10:00:36,241 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:36,241 DEBUG [RS:0;037c58a19ddb:37267-longCompactions-0 {}] compactions.Compactor(225): Compacting 2dac6b2aba484c8ca3588f74e5a36789, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=238, earliestPutTs=1731751231889 2024-11-16T10:00:36,241 DEBUG [RS:0;037c58a19ddb:37267-shortCompactions-0 {}] compactions.Compactor(225): Compacting f234c94f31c345bdacdbdad119738a8c, keycount=150, bloomtype=ROW, size=30.9 K, encoding=NONE, compression=NONE, seqNum=238, earliestPutTs=1731751231889 2024-11-16T10:00:36,241 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:36,242 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:36,242 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:36,242 DEBUG [RS:0;037c58a19ddb:37267-longCompactions-0 {}] compactions.Compactor(225): Compacting e59fe4cb81fc403999f5bb2b146958fc, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=246, earliestPutTs=1731751232526 2024-11-16T10:00:36,242 DEBUG [RS:0;037c58a19ddb:37267-shortCompactions-0 {}] compactions.Compactor(225): Compacting a15a55fc49b74b35997765299aa047dc, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=246, earliestPutTs=1731751232526 2024-11-16T10:00:36,242 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:36,242 DEBUG [RS:0;037c58a19ddb:37267-shortCompactions-0 {}] compactions.Compactor(225): Compacting 889382ab4777414f88c6fe4df51e78eb, keycount=200, bloomtype=ROW, size=39.0 K, encoding=NONE, compression=NONE, seqNum=281, earliestPutTs=1731751234717 2024-11-16T10:00:36,242 DEBUG [RS:0;037c58a19ddb:37267-longCompactions-0 {}] compactions.Compactor(225): Compacting 4c0a41e3bba648e79161d541f37c4939, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=281, earliestPutTs=1731751235344 2024-11-16T10:00:36,242 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:36,242 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:36,243 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:36,243 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:36,243 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:36,243 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:36,243 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:36,243 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:36,244 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:36,244 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:36,244 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:36,244 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:36,244 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:36,245 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:36,245 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:36,245 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:36,245 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:36,246 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:36,246 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:36,246 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:36,246 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:36,246 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:36,247 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:36,247 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:36,247 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:36,247 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:36,248 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:36,248 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:36,248 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:36,248 INFO [RS:0;037c58a19ddb:37267-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=75a5a39ee7bf008864556f27870615b9] 2024-11-16T10:00:36,248 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:36,248 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:36,249 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:36,249 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:36,249 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:36,250 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:36,250 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:36,250 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:36,250 DEBUG [RS:0;037c58a19ddb:37267-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e202411161018498f6a3a49cfa9d301ae0cf02eb9_75a5a39ee7bf008864556f27870615b9 store=[table=TestAcidGuarantees family=A region=75a5a39ee7bf008864556f27870615b9] 2024-11-16T10:00:36,250 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:36,250 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:36,250 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:36,250 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:36,251 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:36,251 INFO [RS:0;037c58a19ddb:37267-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 75a5a39ee7bf008864556f27870615b9#B#compaction#524 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-16T10:00:36,251 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:36,251 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:36,251 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:36,251 DEBUG [RS:0;037c58a19ddb:37267-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/75a5a39ee7bf008864556f27870615b9/.tmp/B/bcb295d9c71e4707b33a2662fd1e76e0 is 50, key is test_row_0/B:col10/1731751235346/Put/seqid=0 2024-11-16T10:00:36,252 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:36,252 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:36,252 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:36,252 DEBUG [RS:0;037c58a19ddb:37267-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e202411161018498f6a3a49cfa9d301ae0cf02eb9_75a5a39ee7bf008864556f27870615b9, store=[table=TestAcidGuarantees family=A region=75a5a39ee7bf008864556f27870615b9] 2024-11-16T10:00:36,252 DEBUG [RS:0;037c58a19ddb:37267-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411161018498f6a3a49cfa9d301ae0cf02eb9_75a5a39ee7bf008864556f27870615b9 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=75a5a39ee7bf008864556f27870615b9] 2024-11-16T10:00:36,252 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:36,252 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:36,252 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:36,253 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:36,253 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:36,253 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:36,254 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:36,254 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:36,254 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:36,254 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:36,254 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:36,255 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:36,255 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:36,255 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37591 is added to blk_1073742443_1619 (size=12949) 2024-11-16T10:00:36,255 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:36,255 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:36,255 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:36,255 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:36,256 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:36,256 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:36,256 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:36,257 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:36,257 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:36,257 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:36,257 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:36,257 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:36,258 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:36,258 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:36,258 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:36,258 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:36,259 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:36,259 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:36,259 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:36,259 DEBUG [RS:0;037c58a19ddb:37267-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/75a5a39ee7bf008864556f27870615b9/.tmp/B/bcb295d9c71e4707b33a2662fd1e76e0 as hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/75a5a39ee7bf008864556f27870615b9/B/bcb295d9c71e4707b33a2662fd1e76e0 2024-11-16T10:00:36,259 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:36,259 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:36,259 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:36,260 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:36,260 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:36,260 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:36,260 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:36,260 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:36,260 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:36,261 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:36,261 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:36,261 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:36,262 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:36,262 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:36,262 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:36,262 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:36,263 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:36,263 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:36,263 INFO [RS:0;037c58a19ddb:37267-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 75a5a39ee7bf008864556f27870615b9/B of 75a5a39ee7bf008864556f27870615b9 into bcb295d9c71e4707b33a2662fd1e76e0(size=12.6 K), total size for store is 12.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-16T10:00:36,263 DEBUG [RS:0;037c58a19ddb:37267-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 75a5a39ee7bf008864556f27870615b9: 2024-11-16T10:00:36,263 INFO [RS:0;037c58a19ddb:37267-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731751217321.75a5a39ee7bf008864556f27870615b9., storeName=75a5a39ee7bf008864556f27870615b9/B, priority=13, startTime=1731751236239; duration=0sec 2024-11-16T10:00:36,263 DEBUG [RS:0;037c58a19ddb:37267-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-16T10:00:36,263 DEBUG [RS:0;037c58a19ddb:37267-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 75a5a39ee7bf008864556f27870615b9:B 2024-11-16T10:00:36,263 DEBUG [RS:0;037c58a19ddb:37267-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-16T10:00:36,264 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:36,264 DEBUG [RS:0;037c58a19ddb:37267-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37149 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-16T10:00:36,264 DEBUG [RS:0;037c58a19ddb:37267-longCompactions-0 {}] regionserver.HStore(1541): 75a5a39ee7bf008864556f27870615b9/C is initiating minor compaction (all files) 2024-11-16T10:00:36,264 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:36,264 INFO [RS:0;037c58a19ddb:37267-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 75a5a39ee7bf008864556f27870615b9/C in TestAcidGuarantees,,1731751217321.75a5a39ee7bf008864556f27870615b9. 2024-11-16T10:00:36,264 INFO [RS:0;037c58a19ddb:37267-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/75a5a39ee7bf008864556f27870615b9/C/c252c96455d64fc29b7b1cda3f9f2f13, hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/75a5a39ee7bf008864556f27870615b9/C/f158f302c9e4447abd35ad6d46fa5b3e, hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/75a5a39ee7bf008864556f27870615b9/C/2f41815c61aa4891bf17f02eb08cfa8e] into tmpdir=hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/75a5a39ee7bf008864556f27870615b9/.tmp, totalSize=36.3 K 2024-11-16T10:00:36,265 DEBUG [RS:0;037c58a19ddb:37267-longCompactions-0 {}] compactions.Compactor(225): Compacting c252c96455d64fc29b7b1cda3f9f2f13, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=238, earliestPutTs=1731751231889 2024-11-16T10:00:36,265 DEBUG [RS:0;037c58a19ddb:37267-longCompactions-0 {}] compactions.Compactor(225): Compacting f158f302c9e4447abd35ad6d46fa5b3e, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=246, earliestPutTs=1731751232526 2024-11-16T10:00:36,265 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:36,266 DEBUG [RS:0;037c58a19ddb:37267-longCompactions-0 {}] compactions.Compactor(225): Compacting 2f41815c61aa4891bf17f02eb08cfa8e, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=281, earliestPutTs=1731751235344 2024-11-16T10:00:36,266 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:36,267 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37591 is added to blk_1073742444_1620 (size=4469) 2024-11-16T10:00:36,268 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:36,268 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:36,269 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:36,269 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:36,269 INFO [RS:0;037c58a19ddb:37267-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 75a5a39ee7bf008864556f27870615b9#A#compaction#523 average throughput is 1.16 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-16T10:00:36,270 DEBUG [RS:0;037c58a19ddb:37267-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/75a5a39ee7bf008864556f27870615b9/.tmp/A/a9799a08cb1d4d1d974bd50f1f7666ee is 175, key is test_row_0/A:col10/1731751235346/Put/seqid=0 2024-11-16T10:00:36,270 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:36,271 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:36,271 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:36,271 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:36,272 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:36,272 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:36,272 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:36,272 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:36,273 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:36,273 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:36,273 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:36,273 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:36,274 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:36,274 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:36,274 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:36,275 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:36,275 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:36,275 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:36,275 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:36,275 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:36,276 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:36,276 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:36,276 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:36,276 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:36,276 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:36,277 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:36,277 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:36,277 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:36,277 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:36,277 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:36,278 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:36,278 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:36,278 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:36,278 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:36,278 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:36,279 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:36,279 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:36,279 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:36,279 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:36,280 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:36,280 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:36,280 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:36,280 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:36,280 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:36,281 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:36,281 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:36,281 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:36,281 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:36,282 INFO [RS:0;037c58a19ddb:37267-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 75a5a39ee7bf008864556f27870615b9#C#compaction#525 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-16T10:00:36,282 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:36,282 DEBUG [RS:0;037c58a19ddb:37267-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/75a5a39ee7bf008864556f27870615b9/.tmp/C/3c4ca7c586824ed4a1983ce0fef4addd is 50, key is test_row_0/C:col10/1731751235346/Put/seqid=0 2024-11-16T10:00:36,282 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:36,283 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:36,283 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:36,283 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:36,283 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:36,283 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:36,283 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:36,284 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:36,284 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:36,284 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:36,284 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:36,285 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:36,285 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:36,285 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:36,285 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:36,286 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:36,286 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:36,286 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:36,286 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:36,286 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:36,287 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:36,287 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:36,287 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:36,287 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:36,287 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:36,288 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:36,288 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:36,288 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:36,288 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:36,288 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:36,288 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:36,289 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:36,289 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:36,289 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:36,290 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:36,290 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:36,290 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37591 is added to blk_1073742445_1621 (size=31903) 2024-11-16T10:00:36,290 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:36,291 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:36,291 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:36,291 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:36,291 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:36,291 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:36,292 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:36,292 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:36,292 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37591 is added to blk_1073742446_1622 (size=12949) 2024-11-16T10:00:36,293 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:36,293 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:36,293 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:36,294 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:36,294 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:36,294 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:36,295 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:36,295 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:36,295 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:36,296 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:36,296 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:36,296 DEBUG [RS:0;037c58a19ddb:37267-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/75a5a39ee7bf008864556f27870615b9/.tmp/A/a9799a08cb1d4d1d974bd50f1f7666ee as hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/75a5a39ee7bf008864556f27870615b9/A/a9799a08cb1d4d1d974bd50f1f7666ee 2024-11-16T10:00:36,296 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:36,296 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:36,297 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:36,297 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:36,297 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:36,297 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:36,298 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:36,298 DEBUG [RS:0;037c58a19ddb:37267-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/75a5a39ee7bf008864556f27870615b9/.tmp/C/3c4ca7c586824ed4a1983ce0fef4addd as hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/75a5a39ee7bf008864556f27870615b9/C/3c4ca7c586824ed4a1983ce0fef4addd 2024-11-16T10:00:36,298 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:36,298 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:36,298 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:36,299 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:36,299 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:36,299 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:36,299 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:36,299 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:36,299 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:36,300 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:36,300 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:36,300 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:36,301 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:36,301 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:36,301 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:36,301 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:36,301 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:36,302 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:36,302 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:36,302 INFO [RS:0;037c58a19ddb:37267-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 75a5a39ee7bf008864556f27870615b9/A of 75a5a39ee7bf008864556f27870615b9 into a9799a08cb1d4d1d974bd50f1f7666ee(size=31.2 K), total size for store is 31.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-16T10:00:36,302 DEBUG [RS:0;037c58a19ddb:37267-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 75a5a39ee7bf008864556f27870615b9: 2024-11-16T10:00:36,302 INFO [RS:0;037c58a19ddb:37267-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731751217321.75a5a39ee7bf008864556f27870615b9., storeName=75a5a39ee7bf008864556f27870615b9/A, priority=13, startTime=1731751236239; duration=0sec 2024-11-16T10:00:36,302 DEBUG [RS:0;037c58a19ddb:37267-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-16T10:00:36,302 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:36,302 DEBUG [RS:0;037c58a19ddb:37267-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 75a5a39ee7bf008864556f27870615b9:A 2024-11-16T10:00:36,302 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:36,303 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:36,303 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:36,303 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:36,303 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:36,304 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:36,304 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:36,305 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:36,305 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:36,305 INFO [RS:0;037c58a19ddb:37267-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 75a5a39ee7bf008864556f27870615b9/C of 75a5a39ee7bf008864556f27870615b9 into 3c4ca7c586824ed4a1983ce0fef4addd(size=12.6 K), total size for store is 12.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-16T10:00:36,305 DEBUG [RS:0;037c58a19ddb:37267-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 75a5a39ee7bf008864556f27870615b9: 2024-11-16T10:00:36,305 INFO [RS:0;037c58a19ddb:37267-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731751217321.75a5a39ee7bf008864556f27870615b9., storeName=75a5a39ee7bf008864556f27870615b9/C, priority=13, startTime=1731751236239; duration=0sec 2024-11-16T10:00:36,305 DEBUG [RS:0;037c58a19ddb:37267-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-16T10:00:36,305 DEBUG [RS:0;037c58a19ddb:37267-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 75a5a39ee7bf008864556f27870615b9:C 2024-11-16T10:00:36,306 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:36,307 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:36,307 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:36,307 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:36,308 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:36,309 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:36,309 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:36,310 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:36,310 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:36,310 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:36,310 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:36,311 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:36,311 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:36,311 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:36,312 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:36,312 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:36,312 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:36,313 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:36,313 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:36,313 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:36,313 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:36,314 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:36,314 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:36,314 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:36,314 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:36,314 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:36,314 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:36,315 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:36,315 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:36,315 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:36,315 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:36,315 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:36,316 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:36,316 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:36,316 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:36,316 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:36,317 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:36,317 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:36,317 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:36,317 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:36,317 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:36,318 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:36,318 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:36,318 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:36,318 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:36,318 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:36,318 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:36,319 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:36,319 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:36,319 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:36,319 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:36,319 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:36,320 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:36,320 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:36,320 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:36,320 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:36,320 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:36,321 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:36,321 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:36,321 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:36,321 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:36,321 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:36,322 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:36,322 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:36,322 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:36,322 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:36,322 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:36,322 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:36,323 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:36,323 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:36,323 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:36,323 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:36,323 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:36,323 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:36,324 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:36,324 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:36,324 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:36,324 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:36,325 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:36,325 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:36,325 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:36,325 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:36,325 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:36,326 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:36,326 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:36,326 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:36,326 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:36,326 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:36,327 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:36,327 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:36,327 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:36,327 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:36,327 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:36,328 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:36,328 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:36,328 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:36,328 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:36,328 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:36,328 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:36,329 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:36,329 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:36,329 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:36,329 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:36,329 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:36,329 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:36,330 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:36,330 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:36,330 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:36,330 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:36,330 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:36,331 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:36,331 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:36,331 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:36,331 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:36,331 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:36,331 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:36,332 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:36,332 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:36,332 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:36,332 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:36,332 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:36,333 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:36,333 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:36,333 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:36,333 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:36,333 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:36,334 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:36,334 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:36,334 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:36,334 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:36,334 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:36,334 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:36,335 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:36,335 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:36,335 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:36,335 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:36,335 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:36,336 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:36,336 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:36,336 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:36,336 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:36,336 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:36,337 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:36,337 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:36,337 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:36,337 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:36,338 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:36,338 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:36,338 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:36,338 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:36,339 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:36,339 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:36,340 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:36,340 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:36,340 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:36,341 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:36,341 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:36,341 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:36,341 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:36,342 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:36,343 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:36,344 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:36,344 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:36,344 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:36,344 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:36,344 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:36,345 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:36,345 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:36,345 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:36,345 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:36,346 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:36,346 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:36,346 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:36,346 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:36,346 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:36,346 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:36,347 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:36,347 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:36,347 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:36,347 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:36,347 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:36,348 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:36,348 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:36,348 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:36,348 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:36,348 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:36,349 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:36,349 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:36,349 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:36,349 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:36,350 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:36,350 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:36,350 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:36,350 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:36,350 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:36,350 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:36,351 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:36,351 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:36,351 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:36,351 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:36,351 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:36,351 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:36,352 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:36,352 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:36,352 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:36,352 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:36,353 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:36,353 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:36,353 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:36,353 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:36,353 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:36,353 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:36,354 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:36,354 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:36,354 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:36,354 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:36,354 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:36,354 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:36,355 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:36,355 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:36,355 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:36,355 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:36,356 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:36,356 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:36,356 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:36,356 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:36,356 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:36,356 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:36,357 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:36,357 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:36,357 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:36,357 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:36,358 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:36,358 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:36,358 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:36,358 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:36,359 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:36,359 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:36,359 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:36,359 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:36,360 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:36,360 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:36,360 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:36,360 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:36,360 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:36,361 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:36,361 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:36,361 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:36,361 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:36,361 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:36,362 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:36,362 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:36,362 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:36,362 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:36,363 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:36,363 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:36,363 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:36,363 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:36,363 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:36,364 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:36,364 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:36,364 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:36,364 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:36,364 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:36,365 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:36,365 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:36,365 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:36,365 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:36,365 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:36,366 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:36,366 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:36,366 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:36,366 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:36,366 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:36,366 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:36,367 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:36,367 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:36,367 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:36,367 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:36,367 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:36,367 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:36,368 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:36,368 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:36,368 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:36,368 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:36,369 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:36,369 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:36,369 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:36,369 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:36,369 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:36,369 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:36,370 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:36,370 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:36,370 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:36,370 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:36,370 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:36,371 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:36,371 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:36,371 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:36,372 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:36,372 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:36,372 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:36,373 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:36,373 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:36,374 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:36,374 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:36,374 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:36,375 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:36,375 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:36,374 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:36,375 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:36,375 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:36,376 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:36,376 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:36,376 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:36,376 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:36,377 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:36,377 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:36,377 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:36,377 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:36,378 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:36,378 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:36,378 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:36,379 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:36,379 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:36,379 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:36,379 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:36,379 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:36,380 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:36,380 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:36,380 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:36,380 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:36,381 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:36,381 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:36,381 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:36,381 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:36,381 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:36,382 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:36,382 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:36,382 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:36,382 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:36,382 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:36,383 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:36,383 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:36,383 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:36,383 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:36,383 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:36,384 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:36,384 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:36,384 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:36,384 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:36,385 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:36,385 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:36,385 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:36,385 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:36,385 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:36,385 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:36,385 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:36,386 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:36,386 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:36,386 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:36,386 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:36,386 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:36,386 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:36,387 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:36,387 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:36,387 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:36,387 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:36,387 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:36,388 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:36,388 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:36,388 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:36,388 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:36,388 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:36,388 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:36,389 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:36,389 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:36,389 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:36,389 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:36,390 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:36,390 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:36,390 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:36,390 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:36,391 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:36,391 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:36,391 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:36,391 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:36,391 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:36,392 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:36,392 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:36,392 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:36,392 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:36,392 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:36,393 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:36,393 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:36,393 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:36,393 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:36,393 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:36,394 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:36,394 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:36,394 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:36,394 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:36,394 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:36,395 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:36,395 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:36,395 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:36,395 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:36,395 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:36,396 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:36,396 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:36,396 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:36,396 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:36,396 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:36,396 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:36,397 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:36,397 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:36,397 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:36,397 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:36,398 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:36,398 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:36,398 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:36,398 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:36,398 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:36,399 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:36,399 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:36,399 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:36,399 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:36,399 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:36,399 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:36,400 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:36,400 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:36,400 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:36,400 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:36,401 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:36,401 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:36,401 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:36,401 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:36,401 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:36,402 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:36,402 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:36,402 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:36,402 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:36,403 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:36,403 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:36,403 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:36,403 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:36,404 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:36,404 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:36,405 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:36,405 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:36,406 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:36,406 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:36,406 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:36,406 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:36,407 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:36,407 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:36,407 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:36,407 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:36,408 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:36,408 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:36,408 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:36,408 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:36,408 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:36,408 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:36,409 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:36,409 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:36,409 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:36,409 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:36,410 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:36,410 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:36,410 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:36,410 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:36,410 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:36,411 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:36,411 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:36,411 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:36,411 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:36,412 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:36,412 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:36,412 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:36,413 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:36,413 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:36,413 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:36,413 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:36,414 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:36,414 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:36,414 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:36,414 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:36,414 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:36,414 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:36,415 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:36,415 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:36,415 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:36,415 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:36,416 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:36,416 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:36,416 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:36,416 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:36,416 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:36,417 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:36,417 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:36,417 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:36,417 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:36,418 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:36,418 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:36,418 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:36,418 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:36,418 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:36,418 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:36,419 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:36,419 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:36,419 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:36,419 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:36,419 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:36,419 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:36,420 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:36,420 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:36,420 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:36,420 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:36,420 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:36,421 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:36,421 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:36,421 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:36,421 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:36,421 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:36,422 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:36,422 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:36,422 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:36,422 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:36,422 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:36,422 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:36,423 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:36,423 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:36,423 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:36,423 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:36,424 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:36,424 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:36,424 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:36,424 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:36,424 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:36,424 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:36,425 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:36,425 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:36,425 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:36,425 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:36,425 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:36,426 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:36,426 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:36,426 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:36,426 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:36,426 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:36,426 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:36,427 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:36,427 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:36,427 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:36,427 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:36,427 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:36,428 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:36,428 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:36,428 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:36,429 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:36,429 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:36,429 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:36,429 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:36,429 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:36,429 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:36,430 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:36,430 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:36,430 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:36,430 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:36,430 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:36,431 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:36,431 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:36,431 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:36,431 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:36,431 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:36,432 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:36,432 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:36,432 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:36,432 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:36,432 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:36,433 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:36,433 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:36,433 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:36,433 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:36,433 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:36,434 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:36,434 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:36,434 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:36,434 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:36,434 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:36,434 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:36,435 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:36,435 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:36,435 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:36,435 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:36,435 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:36,436 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:36,437 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:36,437 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:36,438 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:36,438 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:36,438 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:36,438 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:36,439 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:36,439 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:36,439 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:36,440 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:36,440 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:36,440 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:36,440 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:36,440 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:36,440 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:36,441 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:36,441 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:36,441 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:36,441 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:36,441 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:36,442 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:36,442 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:36,442 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:36,442 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:36,443 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:36,443 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:36,443 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:36,443 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:36,444 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:36,444 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:36,444 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:36,445 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:36,445 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:36,445 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:36,445 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:36,445 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:36,445 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:36,446 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:36,446 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:36,446 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:36,446 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:36,446 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:36,447 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:36,447 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:36,447 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:36,447 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:36,447 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:36,448 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:36,448 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:36,448 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:36,448 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:36,448 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:36,449 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:36,449 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:36,449 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:36,449 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:36,449 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:36,450 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:36,450 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:36,450 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:36,451 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:36,451 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:36,451 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:36,451 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:36,451 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:36,451 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:36,452 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:36,452 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:36,452 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:36,452 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:36,452 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:36,453 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:36,453 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:36,453 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:36,453 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:36,453 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:36,453 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:36,454 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:36,454 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:36,454 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:36,454 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:36,455 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:36,455 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:36,455 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:36,455 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:36,455 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:36,456 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:36,456 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:36,456 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:36,456 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:36,456 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:36,456 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:36,457 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:36,457 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:36,457 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:36,457 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:36,458 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:36,458 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:36,458 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:36,458 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:36,459 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:36,459 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:36,459 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:36,459 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:36,459 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:36,460 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:36,460 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:36,460 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:36,461 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:36,461 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:36,461 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:36,461 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:36,462 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:36,462 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:36,462 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:36,462 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:36,462 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:36,462 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:36,463 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:36,463 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:36,463 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:36,464 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:36,464 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:36,464 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:36,464 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:36,464 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:36,464 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:36,465 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:36,465 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:36,465 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:36,465 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:36,465 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:36,466 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:36,466 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:36,466 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:36,466 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:36,466 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:36,466 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:36,467 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:36,467 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:36,467 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:36,467 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:36,467 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:36,468 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:36,468 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:36,469 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:36,469 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:36,469 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:36,470 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:36,470 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:36,470 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:36,471 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:36,471 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:36,472 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:36,472 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:36,472 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:36,472 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:36,472 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:36,473 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:36,473 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:36,473 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:36,473 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:36,473 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:36,474 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:36,474 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:36,474 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:36,475 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:36,475 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:36,475 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:36,475 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:36,476 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:36,476 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:36,476 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:36,476 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:36,477 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:36,477 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:36,477 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:36,477 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:36,478 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:36,478 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:36,479 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:36,479 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:36,479 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:36,479 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:36,480 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:36,480 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:36,480 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:36,480 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:36,481 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:36,481 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:36,481 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:36,481 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:36,481 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:36,482 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:36,482 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:36,482 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:36,482 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:36,482 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:36,483 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:36,483 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:36,483 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:36,483 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:36,484 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:36,484 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:36,484 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:36,484 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:36,485 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:36,485 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:36,485 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:36,485 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:36,485 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:36,486 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:36,486 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:36,486 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:36,486 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:36,486 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:36,486 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:36,487 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:36,487 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:36,487 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:36,487 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:36,488 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:36,488 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:36,488 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:36,488 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:36,488 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:36,488 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:36,489 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:36,489 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:36,489 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:36,489 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:36,489 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:36,490 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:36,490 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:36,490 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:36,490 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:36,490 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:36,490 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:36,491 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:36,491 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:36,491 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:36,491 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:36,492 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:36,492 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:36,492 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:36,492 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:36,492 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:36,493 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:36,493 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:36,493 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:36,493 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:36,494 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:36,494 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:36,494 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:36,494 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:36,495 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:36,495 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:36,495 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:36,495 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:36,495 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:36,495 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:36,496 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:36,496 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:36,496 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:36,496 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:36,496 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:36,497 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:36,497 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:36,497 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:36,497 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:36,497 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:36,497 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:36,498 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:36,498 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:36,498 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:36,498 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:36,499 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:36,499 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:36,499 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:36,499 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:36,499 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:36,499 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:36,500 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:36,500 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:36,500 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:36,500 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:36,500 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:36,501 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:36,501 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:36,501 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:36,501 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:36,501 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:36,502 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:36,502 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:36,502 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:36,503 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:36,504 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:36,505 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:36,505 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:36,506 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:36,507 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:36,507 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:36,509 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:36,509 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:36,510 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:36,511 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:36,512 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:36,512 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] regionserver.HRegion(8855): Flush requested on 75a5a39ee7bf008864556f27870615b9 2024-11-16T10:00:36,512 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-16T10:00:36,513 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:36,513 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 75a5a39ee7bf008864556f27870615b9 3/3 column families, dataSize=53.67 KB heapSize=141.38 KB 2024-11-16T10:00:36,522 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111663435f957f214c5bb68128e8ac8bcb5c_75a5a39ee7bf008864556f27870615b9 is 50, key is test_row_0/A:col10/1731751236511/Put/seqid=0 2024-11-16T10:00:36,533 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37877 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=160 2024-11-16T10:00:36,533 INFO [RPCClient-NioEventLoopGroup-4-12 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-16T10:00:36,534 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37877 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-16T10:00:36,535 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37877 {}] procedure2.ProcedureExecutor(1139): Stored pid=162, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=162, table=TestAcidGuarantees 2024-11-16T10:00:36,536 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37877 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=162 2024-11-16T10:00:36,536 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=162, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=162, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-16T10:00:36,537 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=162, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=162, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-16T10:00:36,537 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=163, ppid=162, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-16T10:00:36,545 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37591 is added to blk_1073742448_1624 (size=27698) 2024-11-16T10:00:36,546 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=75a5a39ee7bf008864556f27870615b9, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T10:00:36,547 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] ipc.CallRunner(138): callId: 99 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34430 deadline: 1731751296543, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=75a5a39ee7bf008864556f27870615b9, server=037c58a19ddb,37267,1731751077844 2024-11-16T10:00:36,547 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731751217321.75a5a39ee7bf008864556f27870615b9., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 , the old value is region=TestAcidGuarantees,,1731751217321.75a5a39ee7bf008864556f27870615b9., hostname=037c58a19ddb,37267,1731751077844, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=75a5a39ee7bf008864556f27870615b9, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:00:36,547 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731751217321.75a5a39ee7bf008864556f27870615b9., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=75a5a39ee7bf008864556f27870615b9, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:00:36,547 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731751217321.75a5a39ee7bf008864556f27870615b9., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 because the exception is null or not the one we care about 2024-11-16T10:00:36,550 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=75a5a39ee7bf008864556f27870615b9, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T10:00:36,550 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=75a5a39ee7bf008864556f27870615b9, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T10:00:36,550 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] ipc.CallRunner(138): callId: 92 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34428 deadline: 1731751296544, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=75a5a39ee7bf008864556f27870615b9, server=037c58a19ddb,37267,1731751077844 2024-11-16T10:00:36,550 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] ipc.CallRunner(138): callId: 96 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34454 deadline: 1731751296546, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=75a5a39ee7bf008864556f27870615b9, server=037c58a19ddb,37267,1731751077844 2024-11-16T10:00:36,550 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=75a5a39ee7bf008864556f27870615b9, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T10:00:36,550 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731751217321.75a5a39ee7bf008864556f27870615b9., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 , the old value is region=TestAcidGuarantees,,1731751217321.75a5a39ee7bf008864556f27870615b9., hostname=037c58a19ddb,37267,1731751077844, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=75a5a39ee7bf008864556f27870615b9, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:00:36,550 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731751217321.75a5a39ee7bf008864556f27870615b9., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 , the old value is region=TestAcidGuarantees,,1731751217321.75a5a39ee7bf008864556f27870615b9., hostname=037c58a19ddb,37267,1731751077844, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=75a5a39ee7bf008864556f27870615b9, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:00:36,550 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] ipc.CallRunner(138): callId: 97 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34392 deadline: 1731751296547, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=75a5a39ee7bf008864556f27870615b9, server=037c58a19ddb,37267,1731751077844 2024-11-16T10:00:36,550 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731751217321.75a5a39ee7bf008864556f27870615b9., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=75a5a39ee7bf008864556f27870615b9, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:00:36,550 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731751217321.75a5a39ee7bf008864556f27870615b9., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=75a5a39ee7bf008864556f27870615b9, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:00:36,550 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731751217321.75a5a39ee7bf008864556f27870615b9., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 because the exception is null or not the one we care about 2024-11-16T10:00:36,550 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731751217321.75a5a39ee7bf008864556f27870615b9., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 because the exception is null or not the one we care about 2024-11-16T10:00:36,551 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731751217321.75a5a39ee7bf008864556f27870615b9., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 , the old value is region=TestAcidGuarantees,,1731751217321.75a5a39ee7bf008864556f27870615b9., hostname=037c58a19ddb,37267,1731751077844, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=75a5a39ee7bf008864556f27870615b9, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:00:36,551 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731751217321.75a5a39ee7bf008864556f27870615b9., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=75a5a39ee7bf008864556f27870615b9, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:00:36,551 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731751217321.75a5a39ee7bf008864556f27870615b9., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 because the exception is null or not the one we care about 2024-11-16T10:00:36,643 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37877 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=162 2024-11-16T10:00:36,656 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=75a5a39ee7bf008864556f27870615b9, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T10:00:36,656 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] ipc.CallRunner(138): callId: 100 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34430 deadline: 1731751296653, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=75a5a39ee7bf008864556f27870615b9, server=037c58a19ddb,37267,1731751077844 2024-11-16T10:00:36,656 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=75a5a39ee7bf008864556f27870615b9, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T10:00:36,656 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] ipc.CallRunner(138): callId: 93 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34428 deadline: 1731751296653, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=75a5a39ee7bf008864556f27870615b9, server=037c58a19ddb,37267,1731751077844 2024-11-16T10:00:36,656 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731751217321.75a5a39ee7bf008864556f27870615b9., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 , the old value is region=TestAcidGuarantees,,1731751217321.75a5a39ee7bf008864556f27870615b9., hostname=037c58a19ddb,37267,1731751077844, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=75a5a39ee7bf008864556f27870615b9, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:00:36,656 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731751217321.75a5a39ee7bf008864556f27870615b9., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=75a5a39ee7bf008864556f27870615b9, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:00:36,656 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731751217321.75a5a39ee7bf008864556f27870615b9., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 because the exception is null or not the one we care about 2024-11-16T10:00:36,656 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=75a5a39ee7bf008864556f27870615b9, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T10:00:36,656 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] ipc.CallRunner(138): callId: 97 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34454 deadline: 1731751296653, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=75a5a39ee7bf008864556f27870615b9, server=037c58a19ddb,37267,1731751077844 2024-11-16T10:00:36,656 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731751217321.75a5a39ee7bf008864556f27870615b9., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 , the old value is region=TestAcidGuarantees,,1731751217321.75a5a39ee7bf008864556f27870615b9., hostname=037c58a19ddb,37267,1731751077844, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=75a5a39ee7bf008864556f27870615b9, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:00:36,656 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731751217321.75a5a39ee7bf008864556f27870615b9., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=75a5a39ee7bf008864556f27870615b9, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:00:36,656 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731751217321.75a5a39ee7bf008864556f27870615b9., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 because the exception is null or not the one we care about 2024-11-16T10:00:36,656 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=75a5a39ee7bf008864556f27870615b9, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T10:00:36,656 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731751217321.75a5a39ee7bf008864556f27870615b9., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 , the old value is region=TestAcidGuarantees,,1731751217321.75a5a39ee7bf008864556f27870615b9., hostname=037c58a19ddb,37267,1731751077844, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=75a5a39ee7bf008864556f27870615b9, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:00:36,657 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731751217321.75a5a39ee7bf008864556f27870615b9., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=75a5a39ee7bf008864556f27870615b9, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:00:36,657 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731751217321.75a5a39ee7bf008864556f27870615b9., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 because the exception is null or not the one we care about 2024-11-16T10:00:36,657 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] ipc.CallRunner(138): callId: 98 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34392 deadline: 1731751296653, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=75a5a39ee7bf008864556f27870615b9, server=037c58a19ddb,37267,1731751077844 2024-11-16T10:00:36,657 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731751217321.75a5a39ee7bf008864556f27870615b9., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 , the old value is region=TestAcidGuarantees,,1731751217321.75a5a39ee7bf008864556f27870615b9., hostname=037c58a19ddb,37267,1731751077844, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=75a5a39ee7bf008864556f27870615b9, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:00:36,657 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731751217321.75a5a39ee7bf008864556f27870615b9., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=75a5a39ee7bf008864556f27870615b9, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:00:36,657 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731751217321.75a5a39ee7bf008864556f27870615b9., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 because the exception is null or not the one we care about 2024-11-16T10:00:36,689 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=37267 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=163 2024-11-16T10:00:36,689 DEBUG [RS_FLUSH_OPERATIONS-regionserver/037c58a19ddb:0-1 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731751217321.75a5a39ee7bf008864556f27870615b9. 2024-11-16T10:00:36,690 DEBUG [RS_FLUSH_OPERATIONS-regionserver/037c58a19ddb:0-1 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731751217321.75a5a39ee7bf008864556f27870615b9. as already flushing 2024-11-16T10:00:36,690 DEBUG [RS_FLUSH_OPERATIONS-regionserver/037c58a19ddb:0-1 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731751217321.75a5a39ee7bf008864556f27870615b9. 2024-11-16T10:00:36,690 ERROR [RS_FLUSH_OPERATIONS-regionserver/037c58a19ddb:0-1 {event_type=RS_FLUSH_REGIONS, pid=163}] handler.RSProcedureHandler(58): pid=163 java.io.IOException: Unable to complete flush {ENCODED => 75a5a39ee7bf008864556f27870615b9, NAME => 'TestAcidGuarantees,,1731751217321.75a5a39ee7bf008864556f27870615b9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T10:00:36,690 DEBUG [RS_FLUSH_OPERATIONS-regionserver/037c58a19ddb:0-1 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=163 java.io.IOException: Unable to complete flush {ENCODED => 75a5a39ee7bf008864556f27870615b9, NAME => 'TestAcidGuarantees,,1731751217321.75a5a39ee7bf008864556f27870615b9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T10:00:36,690 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37877 {}] master.HMaster(4177): Remote procedure failed, pid=163 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 75a5a39ee7bf008864556f27870615b9, NAME => 'TestAcidGuarantees,,1731751217321.75a5a39ee7bf008864556f27870615b9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 75a5a39ee7bf008864556f27870615b9, NAME => 'TestAcidGuarantees,,1731751217321.75a5a39ee7bf008864556f27870615b9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T10:00:36,842 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=37267 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=163 2024-11-16T10:00:36,842 DEBUG [RS_FLUSH_OPERATIONS-regionserver/037c58a19ddb:0-2 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731751217321.75a5a39ee7bf008864556f27870615b9. 2024-11-16T10:00:36,842 DEBUG [RS_FLUSH_OPERATIONS-regionserver/037c58a19ddb:0-2 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731751217321.75a5a39ee7bf008864556f27870615b9. as already flushing 2024-11-16T10:00:36,842 DEBUG [RS_FLUSH_OPERATIONS-regionserver/037c58a19ddb:0-2 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731751217321.75a5a39ee7bf008864556f27870615b9. 2024-11-16T10:00:36,842 ERROR [RS_FLUSH_OPERATIONS-regionserver/037c58a19ddb:0-2 {event_type=RS_FLUSH_REGIONS, pid=163}] handler.RSProcedureHandler(58): pid=163 java.io.IOException: Unable to complete flush {ENCODED => 75a5a39ee7bf008864556f27870615b9, NAME => 'TestAcidGuarantees,,1731751217321.75a5a39ee7bf008864556f27870615b9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T10:00:36,843 DEBUG [RS_FLUSH_OPERATIONS-regionserver/037c58a19ddb:0-2 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=163 java.io.IOException: Unable to complete flush {ENCODED => 75a5a39ee7bf008864556f27870615b9, NAME => 'TestAcidGuarantees,,1731751217321.75a5a39ee7bf008864556f27870615b9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T10:00:36,843 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37877 {}] master.HMaster(4177): Remote procedure failed, pid=163 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 75a5a39ee7bf008864556f27870615b9, NAME => 'TestAcidGuarantees,,1731751217321.75a5a39ee7bf008864556f27870615b9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 75a5a39ee7bf008864556f27870615b9, NAME => 'TestAcidGuarantees,,1731751217321.75a5a39ee7bf008864556f27870615b9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T10:00:36,853 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37877 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=162 2024-11-16T10:00:36,864 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=75a5a39ee7bf008864556f27870615b9, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T10:00:36,864 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] ipc.CallRunner(138): callId: 94 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34428 deadline: 1731751296862, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=75a5a39ee7bf008864556f27870615b9, server=037c58a19ddb,37267,1731751077844 2024-11-16T10:00:36,864 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=75a5a39ee7bf008864556f27870615b9, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T10:00:36,864 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] ipc.CallRunner(138): callId: 101 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34430 deadline: 1731751296862, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=75a5a39ee7bf008864556f27870615b9, server=037c58a19ddb,37267,1731751077844 2024-11-16T10:00:36,864 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731751217321.75a5a39ee7bf008864556f27870615b9., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 , the old value is region=TestAcidGuarantees,,1731751217321.75a5a39ee7bf008864556f27870615b9., hostname=037c58a19ddb,37267,1731751077844, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=75a5a39ee7bf008864556f27870615b9, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:00:36,864 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731751217321.75a5a39ee7bf008864556f27870615b9., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=75a5a39ee7bf008864556f27870615b9, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:00:36,865 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731751217321.75a5a39ee7bf008864556f27870615b9., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 because the exception is null or not the one we care about 2024-11-16T10:00:36,865 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731751217321.75a5a39ee7bf008864556f27870615b9., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 , the old value is region=TestAcidGuarantees,,1731751217321.75a5a39ee7bf008864556f27870615b9., hostname=037c58a19ddb,37267,1731751077844, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=75a5a39ee7bf008864556f27870615b9, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:00:36,865 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731751217321.75a5a39ee7bf008864556f27870615b9., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=75a5a39ee7bf008864556f27870615b9, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:00:36,865 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731751217321.75a5a39ee7bf008864556f27870615b9., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 because the exception is null or not the one we care about 2024-11-16T10:00:36,866 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=75a5a39ee7bf008864556f27870615b9, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T10:00:36,866 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] ipc.CallRunner(138): callId: 99 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34392 deadline: 1731751296863, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=75a5a39ee7bf008864556f27870615b9, server=037c58a19ddb,37267,1731751077844 2024-11-16T10:00:36,866 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=75a5a39ee7bf008864556f27870615b9, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T10:00:36,866 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731751217321.75a5a39ee7bf008864556f27870615b9., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 , the old value is region=TestAcidGuarantees,,1731751217321.75a5a39ee7bf008864556f27870615b9., hostname=037c58a19ddb,37267,1731751077844, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=75a5a39ee7bf008864556f27870615b9, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:00:36,866 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731751217321.75a5a39ee7bf008864556f27870615b9., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=75a5a39ee7bf008864556f27870615b9, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:00:36,866 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731751217321.75a5a39ee7bf008864556f27870615b9., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 because the exception is null or not the one we care about 2024-11-16T10:00:36,866 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] ipc.CallRunner(138): callId: 98 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34454 deadline: 1731751296863, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=75a5a39ee7bf008864556f27870615b9, server=037c58a19ddb,37267,1731751077844 2024-11-16T10:00:36,866 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731751217321.75a5a39ee7bf008864556f27870615b9., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 , the old value is region=TestAcidGuarantees,,1731751217321.75a5a39ee7bf008864556f27870615b9., hostname=037c58a19ddb,37267,1731751077844, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=75a5a39ee7bf008864556f27870615b9, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:00:36,866 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731751217321.75a5a39ee7bf008864556f27870615b9., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=75a5a39ee7bf008864556f27870615b9, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:00:36,866 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731751217321.75a5a39ee7bf008864556f27870615b9., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 because the exception is null or not the one we care about 2024-11-16T10:00:36,886 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=75a5a39ee7bf008864556f27870615b9, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T10:00:36,886 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] ipc.CallRunner(138): callId: 94 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34412 deadline: 1731751296883, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=75a5a39ee7bf008864556f27870615b9, server=037c58a19ddb,37267,1731751077844 2024-11-16T10:00:36,886 WARN [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRpcRetryingCaller(177): Call to 037c58a19ddb,37267,1731751077844 for 'test_row_1' in 75a5a39ee7bf008864556f27870615b9 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2167 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=75a5a39ee7bf008864556f27870615b9, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor44.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=75a5a39ee7bf008864556f27870615b9, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-16T10:00:36,887 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731751217321.75a5a39ee7bf008864556f27870615b9., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 , the old value is region=TestAcidGuarantees,,1731751217321.75a5a39ee7bf008864556f27870615b9., hostname=037c58a19ddb,37267,1731751077844, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=75a5a39ee7bf008864556f27870615b9, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:00:36,887 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731751217321.75a5a39ee7bf008864556f27870615b9., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=75a5a39ee7bf008864556f27870615b9, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:00:36,887 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731751217321.75a5a39ee7bf008864556f27870615b9., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 because the exception is null or not the one we care about 2024-11-16T10:00:36,946 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:36,949 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111663435f957f214c5bb68128e8ac8bcb5c_75a5a39ee7bf008864556f27870615b9 to hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111663435f957f214c5bb68128e8ac8bcb5c_75a5a39ee7bf008864556f27870615b9 2024-11-16T10:00:36,950 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/75a5a39ee7bf008864556f27870615b9/.tmp/A/c22cfe4cb7594389be246f1bb4cf7a91, store: [table=TestAcidGuarantees family=A region=75a5a39ee7bf008864556f27870615b9] 2024-11-16T10:00:36,951 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/75a5a39ee7bf008864556f27870615b9/.tmp/A/c22cfe4cb7594389be246f1bb4cf7a91 is 175, key is test_row_0/A:col10/1731751236511/Put/seqid=0 2024-11-16T10:00:36,952 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37591 is added to blk_1073742447_1623 (size=83485) 2024-11-16T10:00:36,995 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=37267 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=163 2024-11-16T10:00:36,995 DEBUG [RS_FLUSH_OPERATIONS-regionserver/037c58a19ddb:0-0 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731751217321.75a5a39ee7bf008864556f27870615b9. 2024-11-16T10:00:36,995 DEBUG [RS_FLUSH_OPERATIONS-regionserver/037c58a19ddb:0-0 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731751217321.75a5a39ee7bf008864556f27870615b9. as already flushing 2024-11-16T10:00:36,995 DEBUG [RS_FLUSH_OPERATIONS-regionserver/037c58a19ddb:0-0 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731751217321.75a5a39ee7bf008864556f27870615b9. 2024-11-16T10:00:36,995 ERROR [RS_FLUSH_OPERATIONS-regionserver/037c58a19ddb:0-0 {event_type=RS_FLUSH_REGIONS, pid=163}] handler.RSProcedureHandler(58): pid=163 java.io.IOException: Unable to complete flush {ENCODED => 75a5a39ee7bf008864556f27870615b9, NAME => 'TestAcidGuarantees,,1731751217321.75a5a39ee7bf008864556f27870615b9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T10:00:36,996 DEBUG [RS_FLUSH_OPERATIONS-regionserver/037c58a19ddb:0-0 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=163 java.io.IOException: Unable to complete flush {ENCODED => 75a5a39ee7bf008864556f27870615b9, NAME => 'TestAcidGuarantees,,1731751217321.75a5a39ee7bf008864556f27870615b9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T10:00:36,996 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37877 {}] master.HMaster(4177): Remote procedure failed, pid=163 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 75a5a39ee7bf008864556f27870615b9, NAME => 'TestAcidGuarantees,,1731751217321.75a5a39ee7bf008864556f27870615b9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 75a5a39ee7bf008864556f27870615b9, NAME => 'TestAcidGuarantees,,1731751217321.75a5a39ee7bf008864556f27870615b9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T10:00:37,148 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=37267 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=163 2024-11-16T10:00:37,148 DEBUG [RS_FLUSH_OPERATIONS-regionserver/037c58a19ddb:0-1 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731751217321.75a5a39ee7bf008864556f27870615b9. 2024-11-16T10:00:37,148 DEBUG [RS_FLUSH_OPERATIONS-regionserver/037c58a19ddb:0-1 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731751217321.75a5a39ee7bf008864556f27870615b9. as already flushing 2024-11-16T10:00:37,148 DEBUG [RS_FLUSH_OPERATIONS-regionserver/037c58a19ddb:0-1 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731751217321.75a5a39ee7bf008864556f27870615b9. 2024-11-16T10:00:37,149 ERROR [RS_FLUSH_OPERATIONS-regionserver/037c58a19ddb:0-1 {event_type=RS_FLUSH_REGIONS, pid=163}] handler.RSProcedureHandler(58): pid=163 java.io.IOException: Unable to complete flush {ENCODED => 75a5a39ee7bf008864556f27870615b9, NAME => 'TestAcidGuarantees,,1731751217321.75a5a39ee7bf008864556f27870615b9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T10:00:37,149 DEBUG [RS_FLUSH_OPERATIONS-regionserver/037c58a19ddb:0-1 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=163 java.io.IOException: Unable to complete flush {ENCODED => 75a5a39ee7bf008864556f27870615b9, NAME => 'TestAcidGuarantees,,1731751217321.75a5a39ee7bf008864556f27870615b9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T10:00:37,149 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37877 {}] master.HMaster(4177): Remote procedure failed, pid=163 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 75a5a39ee7bf008864556f27870615b9, NAME => 'TestAcidGuarantees,,1731751217321.75a5a39ee7bf008864556f27870615b9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 75a5a39ee7bf008864556f27870615b9, NAME => 'TestAcidGuarantees,,1731751217321.75a5a39ee7bf008864556f27870615b9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T10:00:37,163 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37877 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=162 2024-11-16T10:00:37,173 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=75a5a39ee7bf008864556f27870615b9, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T10:00:37,174 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] ipc.CallRunner(138): callId: 95 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34428 deadline: 1731751297172, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=75a5a39ee7bf008864556f27870615b9, server=037c58a19ddb,37267,1731751077844 2024-11-16T10:00:37,174 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=75a5a39ee7bf008864556f27870615b9, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T10:00:37,174 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731751217321.75a5a39ee7bf008864556f27870615b9., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 , the old value is region=TestAcidGuarantees,,1731751217321.75a5a39ee7bf008864556f27870615b9., hostname=037c58a19ddb,37267,1731751077844, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=75a5a39ee7bf008864556f27870615b9, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:00:37,174 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731751217321.75a5a39ee7bf008864556f27870615b9., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=75a5a39ee7bf008864556f27870615b9, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:00:37,174 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] ipc.CallRunner(138): callId: 102 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34430 deadline: 1731751297172, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=75a5a39ee7bf008864556f27870615b9, server=037c58a19ddb,37267,1731751077844 2024-11-16T10:00:37,174 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731751217321.75a5a39ee7bf008864556f27870615b9., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 because the exception is null or not the one we care about 2024-11-16T10:00:37,174 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731751217321.75a5a39ee7bf008864556f27870615b9., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 , the old value is region=TestAcidGuarantees,,1731751217321.75a5a39ee7bf008864556f27870615b9., hostname=037c58a19ddb,37267,1731751077844, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=75a5a39ee7bf008864556f27870615b9, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:00:37,174 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731751217321.75a5a39ee7bf008864556f27870615b9., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=75a5a39ee7bf008864556f27870615b9, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:00:37,174 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731751217321.75a5a39ee7bf008864556f27870615b9., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 because the exception is null or not the one we care about 2024-11-16T10:00:37,175 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=75a5a39ee7bf008864556f27870615b9, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T10:00:37,175 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] ipc.CallRunner(138): callId: 100 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34392 deadline: 1731751297172, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=75a5a39ee7bf008864556f27870615b9, server=037c58a19ddb,37267,1731751077844 2024-11-16T10:00:37,175 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=75a5a39ee7bf008864556f27870615b9, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T10:00:37,176 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] ipc.CallRunner(138): callId: 99 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34454 deadline: 1731751297173, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=75a5a39ee7bf008864556f27870615b9, server=037c58a19ddb,37267,1731751077844 2024-11-16T10:00:37,176 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731751217321.75a5a39ee7bf008864556f27870615b9., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 , the old value is region=TestAcidGuarantees,,1731751217321.75a5a39ee7bf008864556f27870615b9., hostname=037c58a19ddb,37267,1731751077844, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=75a5a39ee7bf008864556f27870615b9, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:00:37,176 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731751217321.75a5a39ee7bf008864556f27870615b9., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=75a5a39ee7bf008864556f27870615b9, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:00:37,176 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731751217321.75a5a39ee7bf008864556f27870615b9., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 because the exception is null or not the one we care about 2024-11-16T10:00:37,176 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731751217321.75a5a39ee7bf008864556f27870615b9., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 , the old value is region=TestAcidGuarantees,,1731751217321.75a5a39ee7bf008864556f27870615b9., hostname=037c58a19ddb,37267,1731751077844, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=75a5a39ee7bf008864556f27870615b9, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:00:37,176 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731751217321.75a5a39ee7bf008864556f27870615b9., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=75a5a39ee7bf008864556f27870615b9, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:00:37,176 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731751217321.75a5a39ee7bf008864556f27870615b9., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 because the exception is null or not the one we care about 2024-11-16T10:00:37,301 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=37267 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=163 2024-11-16T10:00:37,301 DEBUG [RS_FLUSH_OPERATIONS-regionserver/037c58a19ddb:0-2 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731751217321.75a5a39ee7bf008864556f27870615b9. 2024-11-16T10:00:37,301 DEBUG [RS_FLUSH_OPERATIONS-regionserver/037c58a19ddb:0-2 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731751217321.75a5a39ee7bf008864556f27870615b9. as already flushing 2024-11-16T10:00:37,301 DEBUG [RS_FLUSH_OPERATIONS-regionserver/037c58a19ddb:0-2 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731751217321.75a5a39ee7bf008864556f27870615b9. 2024-11-16T10:00:37,301 ERROR [RS_FLUSH_OPERATIONS-regionserver/037c58a19ddb:0-2 {event_type=RS_FLUSH_REGIONS, pid=163}] handler.RSProcedureHandler(58): pid=163 java.io.IOException: Unable to complete flush {ENCODED => 75a5a39ee7bf008864556f27870615b9, NAME => 'TestAcidGuarantees,,1731751217321.75a5a39ee7bf008864556f27870615b9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T10:00:37,302 DEBUG [RS_FLUSH_OPERATIONS-regionserver/037c58a19ddb:0-2 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=163 java.io.IOException: Unable to complete flush {ENCODED => 75a5a39ee7bf008864556f27870615b9, NAME => 'TestAcidGuarantees,,1731751217321.75a5a39ee7bf008864556f27870615b9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T10:00:37,302 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37877 {}] master.HMaster(4177): Remote procedure failed, pid=163 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 75a5a39ee7bf008864556f27870615b9, NAME => 'TestAcidGuarantees,,1731751217321.75a5a39ee7bf008864556f27870615b9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 75a5a39ee7bf008864556f27870615b9, NAME => 'TestAcidGuarantees,,1731751217321.75a5a39ee7bf008864556f27870615b9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T10:00:37,353 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=296, memsize=20.1 K, hasBloomFilter=true, into tmp file hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/75a5a39ee7bf008864556f27870615b9/.tmp/A/c22cfe4cb7594389be246f1bb4cf7a91 2024-11-16T10:00:37,360 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/75a5a39ee7bf008864556f27870615b9/.tmp/B/43a66c20363745609f16e83a6a762ca7 is 50, key is test_row_0/B:col10/1731751236511/Put/seqid=0 2024-11-16T10:00:37,363 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37591 is added to blk_1073742449_1625 (size=12301) 2024-11-16T10:00:37,454 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=37267 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=163 2024-11-16T10:00:37,454 DEBUG [RS_FLUSH_OPERATIONS-regionserver/037c58a19ddb:0-0 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731751217321.75a5a39ee7bf008864556f27870615b9. 2024-11-16T10:00:37,454 DEBUG [RS_FLUSH_OPERATIONS-regionserver/037c58a19ddb:0-0 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731751217321.75a5a39ee7bf008864556f27870615b9. as already flushing 2024-11-16T10:00:37,454 DEBUG [RS_FLUSH_OPERATIONS-regionserver/037c58a19ddb:0-0 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731751217321.75a5a39ee7bf008864556f27870615b9. 2024-11-16T10:00:37,454 ERROR [RS_FLUSH_OPERATIONS-regionserver/037c58a19ddb:0-0 {event_type=RS_FLUSH_REGIONS, pid=163}] handler.RSProcedureHandler(58): pid=163 java.io.IOException: Unable to complete flush {ENCODED => 75a5a39ee7bf008864556f27870615b9, NAME => 'TestAcidGuarantees,,1731751217321.75a5a39ee7bf008864556f27870615b9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T10:00:37,455 DEBUG [RS_FLUSH_OPERATIONS-regionserver/037c58a19ddb:0-0 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=163 java.io.IOException: Unable to complete flush {ENCODED => 75a5a39ee7bf008864556f27870615b9, NAME => 'TestAcidGuarantees,,1731751217321.75a5a39ee7bf008864556f27870615b9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T10:00:37,455 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37877 {}] master.HMaster(4177): Remote procedure failed, pid=163 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 75a5a39ee7bf008864556f27870615b9, NAME => 'TestAcidGuarantees,,1731751217321.75a5a39ee7bf008864556f27870615b9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 75a5a39ee7bf008864556f27870615b9, NAME => 'TestAcidGuarantees,,1731751217321.75a5a39ee7bf008864556f27870615b9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T10:00:37,607 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=37267 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=163 2024-11-16T10:00:37,607 DEBUG [RS_FLUSH_OPERATIONS-regionserver/037c58a19ddb:0-1 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731751217321.75a5a39ee7bf008864556f27870615b9. 2024-11-16T10:00:37,607 DEBUG [RS_FLUSH_OPERATIONS-regionserver/037c58a19ddb:0-1 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731751217321.75a5a39ee7bf008864556f27870615b9. as already flushing 2024-11-16T10:00:37,608 DEBUG [RS_FLUSH_OPERATIONS-regionserver/037c58a19ddb:0-1 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731751217321.75a5a39ee7bf008864556f27870615b9. 2024-11-16T10:00:37,608 ERROR [RS_FLUSH_OPERATIONS-regionserver/037c58a19ddb:0-1 {event_type=RS_FLUSH_REGIONS, pid=163}] handler.RSProcedureHandler(58): pid=163 java.io.IOException: Unable to complete flush {ENCODED => 75a5a39ee7bf008864556f27870615b9, NAME => 'TestAcidGuarantees,,1731751217321.75a5a39ee7bf008864556f27870615b9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T10:00:37,608 DEBUG [RS_FLUSH_OPERATIONS-regionserver/037c58a19ddb:0-1 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=163 java.io.IOException: Unable to complete flush {ENCODED => 75a5a39ee7bf008864556f27870615b9, NAME => 'TestAcidGuarantees,,1731751217321.75a5a39ee7bf008864556f27870615b9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T10:00:37,608 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37877 {}] master.HMaster(4177): Remote procedure failed, pid=163 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 75a5a39ee7bf008864556f27870615b9, NAME => 'TestAcidGuarantees,,1731751217321.75a5a39ee7bf008864556f27870615b9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 75a5a39ee7bf008864556f27870615b9, NAME => 'TestAcidGuarantees,,1731751217321.75a5a39ee7bf008864556f27870615b9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T10:00:37,672 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37877 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=162 2024-11-16T10:00:37,684 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=75a5a39ee7bf008864556f27870615b9, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T10:00:37,684 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] ipc.CallRunner(138): callId: 96 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34428 deadline: 1731751297682, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=75a5a39ee7bf008864556f27870615b9, server=037c58a19ddb,37267,1731751077844 2024-11-16T10:00:37,685 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=75a5a39ee7bf008864556f27870615b9, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T10:00:37,685 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] ipc.CallRunner(138): callId: 103 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34430 deadline: 1731751297682, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=75a5a39ee7bf008864556f27870615b9, server=037c58a19ddb,37267,1731751077844 2024-11-16T10:00:37,685 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731751217321.75a5a39ee7bf008864556f27870615b9., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 , the old value is region=TestAcidGuarantees,,1731751217321.75a5a39ee7bf008864556f27870615b9., hostname=037c58a19ddb,37267,1731751077844, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=75a5a39ee7bf008864556f27870615b9, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:00:37,685 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731751217321.75a5a39ee7bf008864556f27870615b9., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=75a5a39ee7bf008864556f27870615b9, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:00:37,685 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731751217321.75a5a39ee7bf008864556f27870615b9., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 because the exception is null or not the one we care about 2024-11-16T10:00:37,685 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=75a5a39ee7bf008864556f27870615b9, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T10:00:37,685 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] ipc.CallRunner(138): callId: 101 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34392 deadline: 1731751297683, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=75a5a39ee7bf008864556f27870615b9, server=037c58a19ddb,37267,1731751077844 2024-11-16T10:00:37,685 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731751217321.75a5a39ee7bf008864556f27870615b9., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 , the old value is region=TestAcidGuarantees,,1731751217321.75a5a39ee7bf008864556f27870615b9., hostname=037c58a19ddb,37267,1731751077844, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=75a5a39ee7bf008864556f27870615b9, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:00:37,685 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731751217321.75a5a39ee7bf008864556f27870615b9., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=75a5a39ee7bf008864556f27870615b9, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:00:37,685 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731751217321.75a5a39ee7bf008864556f27870615b9., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 because the exception is null or not the one we care about 2024-11-16T10:00:37,685 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731751217321.75a5a39ee7bf008864556f27870615b9., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 , the old value is region=TestAcidGuarantees,,1731751217321.75a5a39ee7bf008864556f27870615b9., hostname=037c58a19ddb,37267,1731751077844, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=75a5a39ee7bf008864556f27870615b9, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:00:37,685 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731751217321.75a5a39ee7bf008864556f27870615b9., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=75a5a39ee7bf008864556f27870615b9, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:00:37,685 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=75a5a39ee7bf008864556f27870615b9, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T10:00:37,685 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731751217321.75a5a39ee7bf008864556f27870615b9., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 because the exception is null or not the one we care about 2024-11-16T10:00:37,685 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] ipc.CallRunner(138): callId: 100 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34454 deadline: 1731751297683, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=75a5a39ee7bf008864556f27870615b9, server=037c58a19ddb,37267,1731751077844 2024-11-16T10:00:37,685 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731751217321.75a5a39ee7bf008864556f27870615b9., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 , the old value is region=TestAcidGuarantees,,1731751217321.75a5a39ee7bf008864556f27870615b9., hostname=037c58a19ddb,37267,1731751077844, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=75a5a39ee7bf008864556f27870615b9, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:00:37,686 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731751217321.75a5a39ee7bf008864556f27870615b9., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=75a5a39ee7bf008864556f27870615b9, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:00:37,686 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731751217321.75a5a39ee7bf008864556f27870615b9., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 because the exception is null or not the one we care about 2024-11-16T10:00:37,760 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=37267 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=163 2024-11-16T10:00:37,760 DEBUG [RS_FLUSH_OPERATIONS-regionserver/037c58a19ddb:0-2 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731751217321.75a5a39ee7bf008864556f27870615b9. 2024-11-16T10:00:37,760 DEBUG [RS_FLUSH_OPERATIONS-regionserver/037c58a19ddb:0-2 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731751217321.75a5a39ee7bf008864556f27870615b9. as already flushing 2024-11-16T10:00:37,760 DEBUG [RS_FLUSH_OPERATIONS-regionserver/037c58a19ddb:0-2 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731751217321.75a5a39ee7bf008864556f27870615b9. 2024-11-16T10:00:37,760 ERROR [RS_FLUSH_OPERATIONS-regionserver/037c58a19ddb:0-2 {event_type=RS_FLUSH_REGIONS, pid=163}] handler.RSProcedureHandler(58): pid=163 java.io.IOException: Unable to complete flush {ENCODED => 75a5a39ee7bf008864556f27870615b9, NAME => 'TestAcidGuarantees,,1731751217321.75a5a39ee7bf008864556f27870615b9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T10:00:37,760 DEBUG [RS_FLUSH_OPERATIONS-regionserver/037c58a19ddb:0-2 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=163 java.io.IOException: Unable to complete flush {ENCODED => 75a5a39ee7bf008864556f27870615b9, NAME => 'TestAcidGuarantees,,1731751217321.75a5a39ee7bf008864556f27870615b9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T10:00:37,761 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37877 {}] master.HMaster(4177): Remote procedure failed, pid=163 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 75a5a39ee7bf008864556f27870615b9, NAME => 'TestAcidGuarantees,,1731751217321.75a5a39ee7bf008864556f27870615b9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 75a5a39ee7bf008864556f27870615b9, NAME => 'TestAcidGuarantees,,1731751217321.75a5a39ee7bf008864556f27870615b9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T10:00:37,763 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=296 (bloomFilter=true), to=hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/75a5a39ee7bf008864556f27870615b9/.tmp/B/43a66c20363745609f16e83a6a762ca7 2024-11-16T10:00:37,769 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/75a5a39ee7bf008864556f27870615b9/.tmp/C/d05779c165b8492fa69eef9f745d8ec7 is 50, key is test_row_0/C:col10/1731751236511/Put/seqid=0 2024-11-16T10:00:37,772 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37591 is added to blk_1073742450_1626 (size=12301) 2024-11-16T10:00:37,913 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=37267 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=163 2024-11-16T10:00:37,913 DEBUG [RS_FLUSH_OPERATIONS-regionserver/037c58a19ddb:0-0 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731751217321.75a5a39ee7bf008864556f27870615b9. 2024-11-16T10:00:37,913 DEBUG [RS_FLUSH_OPERATIONS-regionserver/037c58a19ddb:0-0 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731751217321.75a5a39ee7bf008864556f27870615b9. as already flushing 2024-11-16T10:00:37,913 DEBUG [RS_FLUSH_OPERATIONS-regionserver/037c58a19ddb:0-0 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731751217321.75a5a39ee7bf008864556f27870615b9. 2024-11-16T10:00:37,913 ERROR [RS_FLUSH_OPERATIONS-regionserver/037c58a19ddb:0-0 {event_type=RS_FLUSH_REGIONS, pid=163}] handler.RSProcedureHandler(58): pid=163 java.io.IOException: Unable to complete flush {ENCODED => 75a5a39ee7bf008864556f27870615b9, NAME => 'TestAcidGuarantees,,1731751217321.75a5a39ee7bf008864556f27870615b9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T10:00:37,913 DEBUG [RS_FLUSH_OPERATIONS-regionserver/037c58a19ddb:0-0 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=163 java.io.IOException: Unable to complete flush {ENCODED => 75a5a39ee7bf008864556f27870615b9, NAME => 'TestAcidGuarantees,,1731751217321.75a5a39ee7bf008864556f27870615b9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T10:00:37,914 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37877 {}] master.HMaster(4177): Remote procedure failed, pid=163 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 75a5a39ee7bf008864556f27870615b9, NAME => 'TestAcidGuarantees,,1731751217321.75a5a39ee7bf008864556f27870615b9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 75a5a39ee7bf008864556f27870615b9, NAME => 'TestAcidGuarantees,,1731751217321.75a5a39ee7bf008864556f27870615b9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T10:00:38,067 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=37267 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=163 2024-11-16T10:00:38,067 DEBUG [RS_FLUSH_OPERATIONS-regionserver/037c58a19ddb:0-1 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731751217321.75a5a39ee7bf008864556f27870615b9. 2024-11-16T10:00:38,067 DEBUG [RS_FLUSH_OPERATIONS-regionserver/037c58a19ddb:0-1 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731751217321.75a5a39ee7bf008864556f27870615b9. as already flushing 2024-11-16T10:00:38,067 DEBUG [RS_FLUSH_OPERATIONS-regionserver/037c58a19ddb:0-1 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731751217321.75a5a39ee7bf008864556f27870615b9. 2024-11-16T10:00:38,067 ERROR [RS_FLUSH_OPERATIONS-regionserver/037c58a19ddb:0-1 {event_type=RS_FLUSH_REGIONS, pid=163}] handler.RSProcedureHandler(58): pid=163 java.io.IOException: Unable to complete flush {ENCODED => 75a5a39ee7bf008864556f27870615b9, NAME => 'TestAcidGuarantees,,1731751217321.75a5a39ee7bf008864556f27870615b9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T10:00:38,067 DEBUG [RS_FLUSH_OPERATIONS-regionserver/037c58a19ddb:0-1 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=163 java.io.IOException: Unable to complete flush {ENCODED => 75a5a39ee7bf008864556f27870615b9, NAME => 'TestAcidGuarantees,,1731751217321.75a5a39ee7bf008864556f27870615b9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T10:00:38,068 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37877 {}] master.HMaster(4177): Remote procedure failed, pid=163 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 75a5a39ee7bf008864556f27870615b9, NAME => 'TestAcidGuarantees,,1731751217321.75a5a39ee7bf008864556f27870615b9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 75a5a39ee7bf008864556f27870615b9, NAME => 'TestAcidGuarantees,,1731751217321.75a5a39ee7bf008864556f27870615b9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T10:00:38,173 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=296 (bloomFilter=true), to=hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/75a5a39ee7bf008864556f27870615b9/.tmp/C/d05779c165b8492fa69eef9f745d8ec7 2024-11-16T10:00:38,176 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/75a5a39ee7bf008864556f27870615b9/.tmp/A/c22cfe4cb7594389be246f1bb4cf7a91 as hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/75a5a39ee7bf008864556f27870615b9/A/c22cfe4cb7594389be246f1bb4cf7a91 2024-11-16T10:00:38,180 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/75a5a39ee7bf008864556f27870615b9/A/c22cfe4cb7594389be246f1bb4cf7a91, entries=450, sequenceid=296, filesize=81.5 K 2024-11-16T10:00:38,180 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/75a5a39ee7bf008864556f27870615b9/.tmp/B/43a66c20363745609f16e83a6a762ca7 as hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/75a5a39ee7bf008864556f27870615b9/B/43a66c20363745609f16e83a6a762ca7 2024-11-16T10:00:38,184 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/75a5a39ee7bf008864556f27870615b9/B/43a66c20363745609f16e83a6a762ca7, entries=150, sequenceid=296, filesize=12.0 K 2024-11-16T10:00:38,184 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/75a5a39ee7bf008864556f27870615b9/.tmp/C/d05779c165b8492fa69eef9f745d8ec7 as hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/75a5a39ee7bf008864556f27870615b9/C/d05779c165b8492fa69eef9f745d8ec7 2024-11-16T10:00:38,188 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/75a5a39ee7bf008864556f27870615b9/C/d05779c165b8492fa69eef9f745d8ec7, entries=150, sequenceid=296, filesize=12.0 K 2024-11-16T10:00:38,189 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~158.91 KB/162720, currentSize=140.89 KB/144270 for 75a5a39ee7bf008864556f27870615b9 in 1677ms, sequenceid=296, compaction requested=false 2024-11-16T10:00:38,189 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 75a5a39ee7bf008864556f27870615b9: 2024-11-16T10:00:38,221 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=37267 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=163 2024-11-16T10:00:38,221 DEBUG [RS_FLUSH_OPERATIONS-regionserver/037c58a19ddb:0-2 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731751217321.75a5a39ee7bf008864556f27870615b9. 2024-11-16T10:00:38,221 INFO [RS_FLUSH_OPERATIONS-regionserver/037c58a19ddb:0-2 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.HRegion(2902): Flushing 75a5a39ee7bf008864556f27870615b9 3/3 column families, dataSize=140.89 KB heapSize=369.89 KB 2024-11-16T10:00:38,229 DEBUG [RS_FLUSH_OPERATIONS-regionserver/037c58a19ddb:0-2 {event_type=RS_FLUSH_REGIONS, pid=163}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241116db1130e7b0074eb19883b8a8586a467d_75a5a39ee7bf008864556f27870615b9 is 50, key is test_row_0/A:col10/1731751236545/Put/seqid=0 2024-11-16T10:00:38,237 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37591 is added to blk_1073742451_1627 (size=12454) 2024-11-16T10:00:38,237 DEBUG [RS_FLUSH_OPERATIONS-regionserver/037c58a19ddb:0-2 {event_type=RS_FLUSH_REGIONS, pid=163}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:38,241 INFO [RS_FLUSH_OPERATIONS-regionserver/037c58a19ddb:0-2 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241116db1130e7b0074eb19883b8a8586a467d_75a5a39ee7bf008864556f27870615b9 to hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241116db1130e7b0074eb19883b8a8586a467d_75a5a39ee7bf008864556f27870615b9 2024-11-16T10:00:38,242 DEBUG [RS_FLUSH_OPERATIONS-regionserver/037c58a19ddb:0-2 {event_type=RS_FLUSH_REGIONS, pid=163}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/75a5a39ee7bf008864556f27870615b9/.tmp/A/28b43766060b4a8aa1c7479ce4e97c1a, store: [table=TestAcidGuarantees family=A region=75a5a39ee7bf008864556f27870615b9] 2024-11-16T10:00:38,243 DEBUG [RS_FLUSH_OPERATIONS-regionserver/037c58a19ddb:0-2 {event_type=RS_FLUSH_REGIONS, pid=163}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/75a5a39ee7bf008864556f27870615b9/.tmp/A/28b43766060b4a8aa1c7479ce4e97c1a is 175, key is test_row_0/A:col10/1731751236545/Put/seqid=0 2024-11-16T10:00:38,249 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37591 is added to blk_1073742452_1628 (size=31255) 2024-11-16T10:00:38,250 INFO [RS_FLUSH_OPERATIONS-regionserver/037c58a19ddb:0-2 {event_type=RS_FLUSH_REGIONS, pid=163}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=320, memsize=47.0 K, hasBloomFilter=true, into tmp file hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/75a5a39ee7bf008864556f27870615b9/.tmp/A/28b43766060b4a8aa1c7479ce4e97c1a 2024-11-16T10:00:38,266 DEBUG [RS_FLUSH_OPERATIONS-regionserver/037c58a19ddb:0-2 {event_type=RS_FLUSH_REGIONS, pid=163}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/75a5a39ee7bf008864556f27870615b9/.tmp/B/a6494e0ab0dd42e7885c245e52425bfe is 50, key is test_row_0/B:col10/1731751236545/Put/seqid=0 2024-11-16T10:00:38,273 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37591 is added to blk_1073742453_1629 (size=12301) 2024-11-16T10:00:38,673 INFO [RS_FLUSH_OPERATIONS-regionserver/037c58a19ddb:0-2 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=320 (bloomFilter=true), to=hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/75a5a39ee7bf008864556f27870615b9/.tmp/B/a6494e0ab0dd42e7885c245e52425bfe 2024-11-16T10:00:38,680 DEBUG [RS_FLUSH_OPERATIONS-regionserver/037c58a19ddb:0-2 {event_type=RS_FLUSH_REGIONS, pid=163}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/75a5a39ee7bf008864556f27870615b9/.tmp/C/de57cafc4e144f88a43a19eb66c92da3 is 50, key is test_row_0/C:col10/1731751236545/Put/seqid=0 2024-11-16T10:00:38,683 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37877 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=162 2024-11-16T10:00:38,683 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37591 is added to blk_1073742454_1630 (size=12301) 2024-11-16T10:00:38,696 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] regionserver.HRegion(8855): Flush requested on 75a5a39ee7bf008864556f27870615b9 2024-11-16T10:00:38,696 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731751217321.75a5a39ee7bf008864556f27870615b9. as already flushing 2024-11-16T10:00:38,707 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=75a5a39ee7bf008864556f27870615b9, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T10:00:38,708 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] ipc.CallRunner(138): callId: 104 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34392 deadline: 1731751298702, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=75a5a39ee7bf008864556f27870615b9, server=037c58a19ddb,37267,1731751077844 2024-11-16T10:00:38,708 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=75a5a39ee7bf008864556f27870615b9, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T10:00:38,708 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731751217321.75a5a39ee7bf008864556f27870615b9., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 , the old value is region=TestAcidGuarantees,,1731751217321.75a5a39ee7bf008864556f27870615b9., hostname=037c58a19ddb,37267,1731751077844, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=75a5a39ee7bf008864556f27870615b9, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:00:38,708 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731751217321.75a5a39ee7bf008864556f27870615b9., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=75a5a39ee7bf008864556f27870615b9, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:00:38,708 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] ipc.CallRunner(138): callId: 103 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34454 deadline: 1731751298703, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=75a5a39ee7bf008864556f27870615b9, server=037c58a19ddb,37267,1731751077844 2024-11-16T10:00:38,708 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731751217321.75a5a39ee7bf008864556f27870615b9., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 because the exception is null or not the one we care about 2024-11-16T10:00:38,708 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731751217321.75a5a39ee7bf008864556f27870615b9., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 , the old value is region=TestAcidGuarantees,,1731751217321.75a5a39ee7bf008864556f27870615b9., hostname=037c58a19ddb,37267,1731751077844, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=75a5a39ee7bf008864556f27870615b9, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:00:38,708 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731751217321.75a5a39ee7bf008864556f27870615b9., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=75a5a39ee7bf008864556f27870615b9, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:00:38,708 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731751217321.75a5a39ee7bf008864556f27870615b9., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 because the exception is null or not the one we care about 2024-11-16T10:00:38,712 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=75a5a39ee7bf008864556f27870615b9, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T10:00:38,712 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] ipc.CallRunner(138): callId: 100 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34428 deadline: 1731751298707, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=75a5a39ee7bf008864556f27870615b9, server=037c58a19ddb,37267,1731751077844 2024-11-16T10:00:38,712 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=75a5a39ee7bf008864556f27870615b9, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T10:00:38,712 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] ipc.CallRunner(138): callId: 107 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34430 deadline: 1731751298708, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=75a5a39ee7bf008864556f27870615b9, server=037c58a19ddb,37267,1731751077844 2024-11-16T10:00:38,712 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731751217321.75a5a39ee7bf008864556f27870615b9., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 , the old value is region=TestAcidGuarantees,,1731751217321.75a5a39ee7bf008864556f27870615b9., hostname=037c58a19ddb,37267,1731751077844, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=75a5a39ee7bf008864556f27870615b9, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:00:38,712 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731751217321.75a5a39ee7bf008864556f27870615b9., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=75a5a39ee7bf008864556f27870615b9, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:00:38,712 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731751217321.75a5a39ee7bf008864556f27870615b9., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 because the exception is null or not the one we care about 2024-11-16T10:00:38,712 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731751217321.75a5a39ee7bf008864556f27870615b9., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 , the old value is region=TestAcidGuarantees,,1731751217321.75a5a39ee7bf008864556f27870615b9., hostname=037c58a19ddb,37267,1731751077844, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=75a5a39ee7bf008864556f27870615b9, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:00:38,712 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731751217321.75a5a39ee7bf008864556f27870615b9., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=75a5a39ee7bf008864556f27870615b9, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:00:38,712 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731751217321.75a5a39ee7bf008864556f27870615b9., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 because the exception is null or not the one we care about 2024-11-16T10:00:38,816 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=75a5a39ee7bf008864556f27870615b9, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T10:00:38,816 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] ipc.CallRunner(138): callId: 105 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34392 deadline: 1731751298813, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=75a5a39ee7bf008864556f27870615b9, server=037c58a19ddb,37267,1731751077844 2024-11-16T10:00:38,816 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=75a5a39ee7bf008864556f27870615b9, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T10:00:38,816 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731751217321.75a5a39ee7bf008864556f27870615b9., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 , the old value is region=TestAcidGuarantees,,1731751217321.75a5a39ee7bf008864556f27870615b9., hostname=037c58a19ddb,37267,1731751077844, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=75a5a39ee7bf008864556f27870615b9, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:00:38,816 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] ipc.CallRunner(138): callId: 104 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34454 deadline: 1731751298813, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=75a5a39ee7bf008864556f27870615b9, server=037c58a19ddb,37267,1731751077844 2024-11-16T10:00:38,816 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731751217321.75a5a39ee7bf008864556f27870615b9., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=75a5a39ee7bf008864556f27870615b9, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:00:38,816 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731751217321.75a5a39ee7bf008864556f27870615b9., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 because the exception is null or not the one we care about 2024-11-16T10:00:38,816 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731751217321.75a5a39ee7bf008864556f27870615b9., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 , the old value is region=TestAcidGuarantees,,1731751217321.75a5a39ee7bf008864556f27870615b9., hostname=037c58a19ddb,37267,1731751077844, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=75a5a39ee7bf008864556f27870615b9, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:00:38,816 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731751217321.75a5a39ee7bf008864556f27870615b9., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=75a5a39ee7bf008864556f27870615b9, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:00:38,816 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731751217321.75a5a39ee7bf008864556f27870615b9., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 because the exception is null or not the one we care about 2024-11-16T10:00:38,826 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=75a5a39ee7bf008864556f27870615b9, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T10:00:38,826 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] ipc.CallRunner(138): callId: 101 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34428 deadline: 1731751298823, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=75a5a39ee7bf008864556f27870615b9, server=037c58a19ddb,37267,1731751077844 2024-11-16T10:00:38,826 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=75a5a39ee7bf008864556f27870615b9, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T10:00:38,826 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] ipc.CallRunner(138): callId: 108 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34430 deadline: 1731751298823, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=75a5a39ee7bf008864556f27870615b9, server=037c58a19ddb,37267,1731751077844 2024-11-16T10:00:38,826 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731751217321.75a5a39ee7bf008864556f27870615b9., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 , the old value is region=TestAcidGuarantees,,1731751217321.75a5a39ee7bf008864556f27870615b9., hostname=037c58a19ddb,37267,1731751077844, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=75a5a39ee7bf008864556f27870615b9, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:00:38,826 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731751217321.75a5a39ee7bf008864556f27870615b9., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=75a5a39ee7bf008864556f27870615b9, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:00:38,826 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731751217321.75a5a39ee7bf008864556f27870615b9., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 because the exception is null or not the one we care about 2024-11-16T10:00:38,826 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731751217321.75a5a39ee7bf008864556f27870615b9., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 , the old value is region=TestAcidGuarantees,,1731751217321.75a5a39ee7bf008864556f27870615b9., hostname=037c58a19ddb,37267,1731751077844, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=75a5a39ee7bf008864556f27870615b9, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:00:38,826 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731751217321.75a5a39ee7bf008864556f27870615b9., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=75a5a39ee7bf008864556f27870615b9, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:00:38,826 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731751217321.75a5a39ee7bf008864556f27870615b9., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 because the exception is null or not the one we care about 2024-11-16T10:00:38,915 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=75a5a39ee7bf008864556f27870615b9, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T10:00:38,916 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] ipc.CallRunner(138): callId: 95 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34412 deadline: 1731751298913, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=75a5a39ee7bf008864556f27870615b9, server=037c58a19ddb,37267,1731751077844 2024-11-16T10:00:38,916 WARN [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRpcRetryingCaller(177): Call to 037c58a19ddb,37267,1731751077844 for 'test_row_1' in 75a5a39ee7bf008864556f27870615b9 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4197 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=75a5a39ee7bf008864556f27870615b9, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor44.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=75a5a39ee7bf008864556f27870615b9, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-16T10:00:38,916 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731751217321.75a5a39ee7bf008864556f27870615b9., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 , the old value is region=TestAcidGuarantees,,1731751217321.75a5a39ee7bf008864556f27870615b9., hostname=037c58a19ddb,37267,1731751077844, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=75a5a39ee7bf008864556f27870615b9, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:00:38,916 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731751217321.75a5a39ee7bf008864556f27870615b9., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=75a5a39ee7bf008864556f27870615b9, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:00:38,916 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731751217321.75a5a39ee7bf008864556f27870615b9., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 because the exception is null or not the one we care about 2024-11-16T10:00:39,023 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=75a5a39ee7bf008864556f27870615b9, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T10:00:39,023 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] ipc.CallRunner(138): callId: 106 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34392 deadline: 1731751299023, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=75a5a39ee7bf008864556f27870615b9, server=037c58a19ddb,37267,1731751077844 2024-11-16T10:00:39,023 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=75a5a39ee7bf008864556f27870615b9, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T10:00:39,024 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731751217321.75a5a39ee7bf008864556f27870615b9., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 , the old value is region=TestAcidGuarantees,,1731751217321.75a5a39ee7bf008864556f27870615b9., hostname=037c58a19ddb,37267,1731751077844, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=75a5a39ee7bf008864556f27870615b9, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:00:39,024 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731751217321.75a5a39ee7bf008864556f27870615b9., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=75a5a39ee7bf008864556f27870615b9, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:00:39,024 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] ipc.CallRunner(138): callId: 105 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34454 deadline: 1731751299023, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=75a5a39ee7bf008864556f27870615b9, server=037c58a19ddb,37267,1731751077844 2024-11-16T10:00:39,024 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731751217321.75a5a39ee7bf008864556f27870615b9., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 because the exception is null or not the one we care about 2024-11-16T10:00:39,024 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731751217321.75a5a39ee7bf008864556f27870615b9., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 , the old value is region=TestAcidGuarantees,,1731751217321.75a5a39ee7bf008864556f27870615b9., hostname=037c58a19ddb,37267,1731751077844, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=75a5a39ee7bf008864556f27870615b9, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:00:39,024 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731751217321.75a5a39ee7bf008864556f27870615b9., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=75a5a39ee7bf008864556f27870615b9, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:00:39,024 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731751217321.75a5a39ee7bf008864556f27870615b9., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 because the exception is null or not the one we care about 2024-11-16T10:00:39,036 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=75a5a39ee7bf008864556f27870615b9, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T10:00:39,036 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] ipc.CallRunner(138): callId: 102 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34428 deadline: 1731751299033, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=75a5a39ee7bf008864556f27870615b9, server=037c58a19ddb,37267,1731751077844 2024-11-16T10:00:39,036 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=75a5a39ee7bf008864556f27870615b9, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T10:00:39,037 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] ipc.CallRunner(138): callId: 109 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34430 deadline: 1731751299033, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=75a5a39ee7bf008864556f27870615b9, server=037c58a19ddb,37267,1731751077844 2024-11-16T10:00:39,037 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731751217321.75a5a39ee7bf008864556f27870615b9., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 , the old value is region=TestAcidGuarantees,,1731751217321.75a5a39ee7bf008864556f27870615b9., hostname=037c58a19ddb,37267,1731751077844, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=75a5a39ee7bf008864556f27870615b9, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:00:39,037 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731751217321.75a5a39ee7bf008864556f27870615b9., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=75a5a39ee7bf008864556f27870615b9, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:00:39,037 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731751217321.75a5a39ee7bf008864556f27870615b9., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 because the exception is null or not the one we care about 2024-11-16T10:00:39,037 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731751217321.75a5a39ee7bf008864556f27870615b9., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 , the old value is region=TestAcidGuarantees,,1731751217321.75a5a39ee7bf008864556f27870615b9., hostname=037c58a19ddb,37267,1731751077844, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=75a5a39ee7bf008864556f27870615b9, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:00:39,037 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731751217321.75a5a39ee7bf008864556f27870615b9., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=75a5a39ee7bf008864556f27870615b9, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:00:39,037 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731751217321.75a5a39ee7bf008864556f27870615b9., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 because the exception is null or not the one we care about 2024-11-16T10:00:39,084 INFO [RS_FLUSH_OPERATIONS-regionserver/037c58a19ddb:0-2 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=320 (bloomFilter=true), to=hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/75a5a39ee7bf008864556f27870615b9/.tmp/C/de57cafc4e144f88a43a19eb66c92da3 2024-11-16T10:00:39,087 DEBUG [RS_FLUSH_OPERATIONS-regionserver/037c58a19ddb:0-2 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/75a5a39ee7bf008864556f27870615b9/.tmp/A/28b43766060b4a8aa1c7479ce4e97c1a as hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/75a5a39ee7bf008864556f27870615b9/A/28b43766060b4a8aa1c7479ce4e97c1a 2024-11-16T10:00:39,090 INFO [RS_FLUSH_OPERATIONS-regionserver/037c58a19ddb:0-2 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/75a5a39ee7bf008864556f27870615b9/A/28b43766060b4a8aa1c7479ce4e97c1a, entries=150, sequenceid=320, filesize=30.5 K 2024-11-16T10:00:39,091 DEBUG [RS_FLUSH_OPERATIONS-regionserver/037c58a19ddb:0-2 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/75a5a39ee7bf008864556f27870615b9/.tmp/B/a6494e0ab0dd42e7885c245e52425bfe as hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/75a5a39ee7bf008864556f27870615b9/B/a6494e0ab0dd42e7885c245e52425bfe 2024-11-16T10:00:39,094 INFO [RS_FLUSH_OPERATIONS-regionserver/037c58a19ddb:0-2 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/75a5a39ee7bf008864556f27870615b9/B/a6494e0ab0dd42e7885c245e52425bfe, entries=150, sequenceid=320, filesize=12.0 K 2024-11-16T10:00:39,095 DEBUG [RS_FLUSH_OPERATIONS-regionserver/037c58a19ddb:0-2 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/75a5a39ee7bf008864556f27870615b9/.tmp/C/de57cafc4e144f88a43a19eb66c92da3 as hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/75a5a39ee7bf008864556f27870615b9/C/de57cafc4e144f88a43a19eb66c92da3 2024-11-16T10:00:39,098 INFO [RS_FLUSH_OPERATIONS-regionserver/037c58a19ddb:0-2 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/75a5a39ee7bf008864556f27870615b9/C/de57cafc4e144f88a43a19eb66c92da3, entries=150, sequenceid=320, filesize=12.0 K 2024-11-16T10:00:39,099 INFO [RS_FLUSH_OPERATIONS-regionserver/037c58a19ddb:0-2 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.HRegion(3140): Finished flush of dataSize ~140.89 KB/144270, heapSize ~369.84 KB/378720, currentSize=67.09 KB/68700 for 75a5a39ee7bf008864556f27870615b9 in 877ms, sequenceid=320, compaction requested=true 2024-11-16T10:00:39,099 DEBUG [RS_FLUSH_OPERATIONS-regionserver/037c58a19ddb:0-2 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.HRegion(2603): Flush status journal for 75a5a39ee7bf008864556f27870615b9: 2024-11-16T10:00:39,099 DEBUG [RS_FLUSH_OPERATIONS-regionserver/037c58a19ddb:0-2 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731751217321.75a5a39ee7bf008864556f27870615b9. 2024-11-16T10:00:39,099 DEBUG [RS_FLUSH_OPERATIONS-regionserver/037c58a19ddb:0-2 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=163 2024-11-16T10:00:39,099 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37877 {}] master.HMaster(4169): Remote procedure done, pid=163 2024-11-16T10:00:39,102 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=163, resume processing ppid=162 2024-11-16T10:00:39,102 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=163, ppid=162, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.5630 sec 2024-11-16T10:00:39,104 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=162, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=162, table=TestAcidGuarantees in 2.5680 sec 2024-11-16T10:00:39,336 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] regionserver.HRegion(8855): Flush requested on 75a5a39ee7bf008864556f27870615b9 2024-11-16T10:00:39,336 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-16T10:00:39,336 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 75a5a39ee7bf008864556f27870615b9 3/3 column families, dataSize=80.51 KB heapSize=211.69 KB 2024-11-16T10:00:39,343 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411169de0d3c95d8d4cd69e4c3255e868a303_75a5a39ee7bf008864556f27870615b9 is 50, key is test_row_0/A:col10/1731751239335/Put/seqid=0 2024-11-16T10:00:39,347 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37591 is added to blk_1073742455_1631 (size=17534) 2024-11-16T10:00:39,348 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:39,350 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411169de0d3c95d8d4cd69e4c3255e868a303_75a5a39ee7bf008864556f27870615b9 to hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411169de0d3c95d8d4cd69e4c3255e868a303_75a5a39ee7bf008864556f27870615b9 2024-11-16T10:00:39,351 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/75a5a39ee7bf008864556f27870615b9/.tmp/A/b5de0c395eb44babbf679ef96e17b28b, store: [table=TestAcidGuarantees family=A region=75a5a39ee7bf008864556f27870615b9] 2024-11-16T10:00:39,351 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/75a5a39ee7bf008864556f27870615b9/.tmp/A/b5de0c395eb44babbf679ef96e17b28b is 175, key is test_row_0/A:col10/1731751239335/Put/seqid=0 2024-11-16T10:00:39,355 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37591 is added to blk_1073742456_1632 (size=48639) 2024-11-16T10:00:39,369 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=75a5a39ee7bf008864556f27870615b9, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T10:00:39,369 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] ipc.CallRunner(138): callId: 112 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34454 deadline: 1731751299364, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=75a5a39ee7bf008864556f27870615b9, server=037c58a19ddb,37267,1731751077844 2024-11-16T10:00:39,369 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731751217321.75a5a39ee7bf008864556f27870615b9., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 , the old value is region=TestAcidGuarantees,,1731751217321.75a5a39ee7bf008864556f27870615b9., hostname=037c58a19ddb,37267,1731751077844, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=75a5a39ee7bf008864556f27870615b9, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:00:39,369 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731751217321.75a5a39ee7bf008864556f27870615b9., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=75a5a39ee7bf008864556f27870615b9, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:00:39,369 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731751217321.75a5a39ee7bf008864556f27870615b9., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 because the exception is null or not the one we care about 2024-11-16T10:00:39,372 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=75a5a39ee7bf008864556f27870615b9, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T10:00:39,372 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] ipc.CallRunner(138): callId: 113 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34392 deadline: 1731751299367, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=75a5a39ee7bf008864556f27870615b9, server=037c58a19ddb,37267,1731751077844 2024-11-16T10:00:39,372 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731751217321.75a5a39ee7bf008864556f27870615b9., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 , the old value is region=TestAcidGuarantees,,1731751217321.75a5a39ee7bf008864556f27870615b9., hostname=037c58a19ddb,37267,1731751077844, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=75a5a39ee7bf008864556f27870615b9, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:00:39,372 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731751217321.75a5a39ee7bf008864556f27870615b9., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=75a5a39ee7bf008864556f27870615b9, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:00:39,373 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731751217321.75a5a39ee7bf008864556f27870615b9., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 because the exception is null or not the one we care about 2024-11-16T10:00:39,373 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=75a5a39ee7bf008864556f27870615b9, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T10:00:39,373 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] ipc.CallRunner(138): callId: 107 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34428 deadline: 1731751299368, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=75a5a39ee7bf008864556f27870615b9, server=037c58a19ddb,37267,1731751077844 2024-11-16T10:00:39,373 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=75a5a39ee7bf008864556f27870615b9, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T10:00:39,373 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] ipc.CallRunner(138): callId: 114 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34430 deadline: 1731751299369, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=75a5a39ee7bf008864556f27870615b9, server=037c58a19ddb,37267,1731751077844 2024-11-16T10:00:39,373 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731751217321.75a5a39ee7bf008864556f27870615b9., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 , the old value is region=TestAcidGuarantees,,1731751217321.75a5a39ee7bf008864556f27870615b9., hostname=037c58a19ddb,37267,1731751077844, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=75a5a39ee7bf008864556f27870615b9, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:00:39,373 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731751217321.75a5a39ee7bf008864556f27870615b9., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=75a5a39ee7bf008864556f27870615b9, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:00:39,373 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731751217321.75a5a39ee7bf008864556f27870615b9., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 because the exception is null or not the one we care about 2024-11-16T10:00:39,373 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731751217321.75a5a39ee7bf008864556f27870615b9., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 , the old value is region=TestAcidGuarantees,,1731751217321.75a5a39ee7bf008864556f27870615b9., hostname=037c58a19ddb,37267,1731751077844, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=75a5a39ee7bf008864556f27870615b9, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:00:39,373 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731751217321.75a5a39ee7bf008864556f27870615b9., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=75a5a39ee7bf008864556f27870615b9, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:00:39,373 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731751217321.75a5a39ee7bf008864556f27870615b9., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 because the exception is null or not the one we care about 2024-11-16T10:00:39,476 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=75a5a39ee7bf008864556f27870615b9, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T10:00:39,476 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] ipc.CallRunner(138): callId: 113 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34454 deadline: 1731751299473, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=75a5a39ee7bf008864556f27870615b9, server=037c58a19ddb,37267,1731751077844 2024-11-16T10:00:39,477 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731751217321.75a5a39ee7bf008864556f27870615b9., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 , the old value is region=TestAcidGuarantees,,1731751217321.75a5a39ee7bf008864556f27870615b9., hostname=037c58a19ddb,37267,1731751077844, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=75a5a39ee7bf008864556f27870615b9, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:00:39,477 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731751217321.75a5a39ee7bf008864556f27870615b9., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=75a5a39ee7bf008864556f27870615b9, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:00:39,477 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731751217321.75a5a39ee7bf008864556f27870615b9., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 because the exception is null or not the one we care about 2024-11-16T10:00:39,484 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=75a5a39ee7bf008864556f27870615b9, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T10:00:39,484 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] ipc.CallRunner(138): callId: 114 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34392 deadline: 1731751299482, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=75a5a39ee7bf008864556f27870615b9, server=037c58a19ddb,37267,1731751077844 2024-11-16T10:00:39,489 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731751217321.75a5a39ee7bf008864556f27870615b9., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 , the old value is region=TestAcidGuarantees,,1731751217321.75a5a39ee7bf008864556f27870615b9., hostname=037c58a19ddb,37267,1731751077844, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=75a5a39ee7bf008864556f27870615b9, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:00:39,489 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=75a5a39ee7bf008864556f27870615b9, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T10:00:39,489 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731751217321.75a5a39ee7bf008864556f27870615b9., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=75a5a39ee7bf008864556f27870615b9, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:00:39,489 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] ipc.CallRunner(138): callId: 108 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34428 deadline: 1731751299482, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=75a5a39ee7bf008864556f27870615b9, server=037c58a19ddb,37267,1731751077844 2024-11-16T10:00:39,489 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731751217321.75a5a39ee7bf008864556f27870615b9., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 because the exception is null or not the one we care about 2024-11-16T10:00:39,489 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=75a5a39ee7bf008864556f27870615b9, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T10:00:39,489 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731751217321.75a5a39ee7bf008864556f27870615b9., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 , the old value is region=TestAcidGuarantees,,1731751217321.75a5a39ee7bf008864556f27870615b9., hostname=037c58a19ddb,37267,1731751077844, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=75a5a39ee7bf008864556f27870615b9, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:00:39,489 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731751217321.75a5a39ee7bf008864556f27870615b9., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=75a5a39ee7bf008864556f27870615b9, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:00:39,489 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] ipc.CallRunner(138): callId: 115 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34430 deadline: 1731751299482, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=75a5a39ee7bf008864556f27870615b9, server=037c58a19ddb,37267,1731751077844 2024-11-16T10:00:39,489 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731751217321.75a5a39ee7bf008864556f27870615b9., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 because the exception is null or not the one we care about 2024-11-16T10:00:39,490 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731751217321.75a5a39ee7bf008864556f27870615b9., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 , the old value is region=TestAcidGuarantees,,1731751217321.75a5a39ee7bf008864556f27870615b9., hostname=037c58a19ddb,37267,1731751077844, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=75a5a39ee7bf008864556f27870615b9, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:00:39,490 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731751217321.75a5a39ee7bf008864556f27870615b9., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=75a5a39ee7bf008864556f27870615b9, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:00:39,490 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731751217321.75a5a39ee7bf008864556f27870615b9., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 because the exception is null or not the one we care about 2024-11-16T10:00:39,685 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=75a5a39ee7bf008864556f27870615b9, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T10:00:39,685 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] ipc.CallRunner(138): callId: 114 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34454 deadline: 1731751299683, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=75a5a39ee7bf008864556f27870615b9, server=037c58a19ddb,37267,1731751077844 2024-11-16T10:00:39,685 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731751217321.75a5a39ee7bf008864556f27870615b9., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 , the old value is region=TestAcidGuarantees,,1731751217321.75a5a39ee7bf008864556f27870615b9., hostname=037c58a19ddb,37267,1731751077844, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=75a5a39ee7bf008864556f27870615b9, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:00:39,685 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731751217321.75a5a39ee7bf008864556f27870615b9., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=75a5a39ee7bf008864556f27870615b9, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:00:39,685 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731751217321.75a5a39ee7bf008864556f27870615b9., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 because the exception is null or not the one we care about 2024-11-16T10:00:39,694 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=75a5a39ee7bf008864556f27870615b9, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T10:00:39,695 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] ipc.CallRunner(138): callId: 115 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34392 deadline: 1731751299692, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=75a5a39ee7bf008864556f27870615b9, server=037c58a19ddb,37267,1731751077844 2024-11-16T10:00:39,695 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731751217321.75a5a39ee7bf008864556f27870615b9., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 , the old value is region=TestAcidGuarantees,,1731751217321.75a5a39ee7bf008864556f27870615b9., hostname=037c58a19ddb,37267,1731751077844, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=75a5a39ee7bf008864556f27870615b9, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:00:39,695 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=75a5a39ee7bf008864556f27870615b9, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T10:00:39,695 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731751217321.75a5a39ee7bf008864556f27870615b9., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=75a5a39ee7bf008864556f27870615b9, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:00:39,695 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731751217321.75a5a39ee7bf008864556f27870615b9., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 because the exception is null or not the one we care about 2024-11-16T10:00:39,695 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] ipc.CallRunner(138): callId: 109 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34428 deadline: 1731751299692, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=75a5a39ee7bf008864556f27870615b9, server=037c58a19ddb,37267,1731751077844 2024-11-16T10:00:39,695 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=75a5a39ee7bf008864556f27870615b9, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T10:00:39,695 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] ipc.CallRunner(138): callId: 116 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34430 deadline: 1731751299693, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=75a5a39ee7bf008864556f27870615b9, server=037c58a19ddb,37267,1731751077844 2024-11-16T10:00:39,695 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731751217321.75a5a39ee7bf008864556f27870615b9., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 , the old value is region=TestAcidGuarantees,,1731751217321.75a5a39ee7bf008864556f27870615b9., hostname=037c58a19ddb,37267,1731751077844, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=75a5a39ee7bf008864556f27870615b9, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:00:39,695 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731751217321.75a5a39ee7bf008864556f27870615b9., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=75a5a39ee7bf008864556f27870615b9, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:00:39,695 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731751217321.75a5a39ee7bf008864556f27870615b9., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 because the exception is null or not the one we care about 2024-11-16T10:00:39,695 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731751217321.75a5a39ee7bf008864556f27870615b9., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 , the old value is region=TestAcidGuarantees,,1731751217321.75a5a39ee7bf008864556f27870615b9., hostname=037c58a19ddb,37267,1731751077844, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=75a5a39ee7bf008864556f27870615b9, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:00:39,696 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731751217321.75a5a39ee7bf008864556f27870615b9., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=75a5a39ee7bf008864556f27870615b9, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:00:39,696 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731751217321.75a5a39ee7bf008864556f27870615b9., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 because the exception is null or not the one we care about 2024-11-16T10:00:39,756 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=335, memsize=26.8 K, hasBloomFilter=true, into tmp file hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/75a5a39ee7bf008864556f27870615b9/.tmp/A/b5de0c395eb44babbf679ef96e17b28b 2024-11-16T10:00:39,762 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/75a5a39ee7bf008864556f27870615b9/.tmp/B/c7fd2d10579b4772ab5af7d1b90dd669 is 50, key is test_row_0/B:col10/1731751239335/Put/seqid=0 2024-11-16T10:00:39,766 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37591 is added to blk_1073742457_1633 (size=12301) 2024-11-16T10:00:39,995 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=75a5a39ee7bf008864556f27870615b9, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T10:00:39,996 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] ipc.CallRunner(138): callId: 115 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34454 deadline: 1731751299993, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=75a5a39ee7bf008864556f27870615b9, server=037c58a19ddb,37267,1731751077844 2024-11-16T10:00:39,996 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731751217321.75a5a39ee7bf008864556f27870615b9., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 , the old value is region=TestAcidGuarantees,,1731751217321.75a5a39ee7bf008864556f27870615b9., hostname=037c58a19ddb,37267,1731751077844, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=75a5a39ee7bf008864556f27870615b9, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:00:39,996 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731751217321.75a5a39ee7bf008864556f27870615b9., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=75a5a39ee7bf008864556f27870615b9, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:00:39,996 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731751217321.75a5a39ee7bf008864556f27870615b9., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 because the exception is null or not the one we care about 2024-11-16T10:00:40,006 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=75a5a39ee7bf008864556f27870615b9, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T10:00:40,006 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] ipc.CallRunner(138): callId: 116 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34392 deadline: 1731751300003, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=75a5a39ee7bf008864556f27870615b9, server=037c58a19ddb,37267,1731751077844 2024-11-16T10:00:40,006 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=75a5a39ee7bf008864556f27870615b9, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T10:00:40,006 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731751217321.75a5a39ee7bf008864556f27870615b9., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 , the old value is region=TestAcidGuarantees,,1731751217321.75a5a39ee7bf008864556f27870615b9., hostname=037c58a19ddb,37267,1731751077844, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=75a5a39ee7bf008864556f27870615b9, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:00:40,006 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731751217321.75a5a39ee7bf008864556f27870615b9., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=75a5a39ee7bf008864556f27870615b9, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:00:40,006 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] ipc.CallRunner(138): callId: 117 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34430 deadline: 1731751300003, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=75a5a39ee7bf008864556f27870615b9, server=037c58a19ddb,37267,1731751077844 2024-11-16T10:00:40,006 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731751217321.75a5a39ee7bf008864556f27870615b9., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 because the exception is null or not the one we care about 2024-11-16T10:00:40,006 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731751217321.75a5a39ee7bf008864556f27870615b9., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 , the old value is region=TestAcidGuarantees,,1731751217321.75a5a39ee7bf008864556f27870615b9., hostname=037c58a19ddb,37267,1731751077844, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=75a5a39ee7bf008864556f27870615b9, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:00:40,006 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731751217321.75a5a39ee7bf008864556f27870615b9., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=75a5a39ee7bf008864556f27870615b9, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:00:40,006 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=75a5a39ee7bf008864556f27870615b9, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T10:00:40,006 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731751217321.75a5a39ee7bf008864556f27870615b9., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 because the exception is null or not the one we care about 2024-11-16T10:00:40,006 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] ipc.CallRunner(138): callId: 110 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34428 deadline: 1731751300003, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=75a5a39ee7bf008864556f27870615b9, server=037c58a19ddb,37267,1731751077844 2024-11-16T10:00:40,007 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731751217321.75a5a39ee7bf008864556f27870615b9., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 , the old value is region=TestAcidGuarantees,,1731751217321.75a5a39ee7bf008864556f27870615b9., hostname=037c58a19ddb,37267,1731751077844, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=75a5a39ee7bf008864556f27870615b9, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:00:40,007 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731751217321.75a5a39ee7bf008864556f27870615b9., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=75a5a39ee7bf008864556f27870615b9, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:00:40,007 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731751217321.75a5a39ee7bf008864556f27870615b9., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 because the exception is null or not the one we care about 2024-11-16T10:00:40,167 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=335 (bloomFilter=true), to=hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/75a5a39ee7bf008864556f27870615b9/.tmp/B/c7fd2d10579b4772ab5af7d1b90dd669 2024-11-16T10:00:40,174 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/75a5a39ee7bf008864556f27870615b9/.tmp/C/c97e7346574145ee8de55ce290fdd612 is 50, key is test_row_0/C:col10/1731751239335/Put/seqid=0 2024-11-16T10:00:40,177 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37591 is added to blk_1073742458_1634 (size=12301) 2024-11-16T10:00:40,443 INFO [Thread-2427 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2427. 2024-11-16T10:00:40,443 DEBUG [Thread-2427 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-16T10:00:40,443 DEBUG [Thread-2427 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T10:00:40,443 DEBUG [Thread-2427 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T10:00:40,443 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-16T10:00:40,443 INFO [Thread-2425 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2425. 2024-11-16T10:00:40,443 DEBUG [Thread-2425 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-16T10:00:40,443 DEBUG [Thread-2425 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T10:00:40,443 DEBUG [Thread-2425 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T10:00:40,443 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-16T10:00:40,446 INFO [Thread-2433 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2433. 2024-11-16T10:00:40,446 DEBUG [Thread-2433 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-16T10:00:40,446 DEBUG [Thread-2433 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T10:00:40,446 DEBUG [Thread-2433 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T10:00:40,446 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-16T10:00:40,447 INFO [Thread-2431 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2431. 2024-11-16T10:00:40,447 DEBUG [Thread-2431 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-16T10:00:40,447 DEBUG [Thread-2431 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T10:00:40,447 DEBUG [Thread-2431 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T10:00:40,447 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-16T10:00:40,447 INFO [Thread-2429 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2429. 2024-11-16T10:00:40,447 DEBUG [Thread-2429 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-16T10:00:40,447 DEBUG [Thread-2429 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T10:00:40,448 DEBUG [Thread-2429 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T10:00:40,448 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-16T10:00:40,503 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=75a5a39ee7bf008864556f27870615b9, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T10:00:40,503 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] ipc.CallRunner(138): callId: 116 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34454 deadline: 1731751300503, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=75a5a39ee7bf008864556f27870615b9, server=037c58a19ddb,37267,1731751077844 2024-11-16T10:00:40,503 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731751217321.75a5a39ee7bf008864556f27870615b9., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 , the old value is region=TestAcidGuarantees,,1731751217321.75a5a39ee7bf008864556f27870615b9., hostname=037c58a19ddb,37267,1731751077844, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=75a5a39ee7bf008864556f27870615b9, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:00:40,503 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731751217321.75a5a39ee7bf008864556f27870615b9., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=75a5a39ee7bf008864556f27870615b9, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:00:40,503 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731751217321.75a5a39ee7bf008864556f27870615b9., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 because the exception is null or not the one we care about 2024-11-16T10:00:40,513 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=75a5a39ee7bf008864556f27870615b9, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T10:00:40,513 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37267 {}] ipc.CallRunner(138): callId: 117 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34392 deadline: 1731751300513, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=75a5a39ee7bf008864556f27870615b9, server=037c58a19ddb,37267,1731751077844 2024-11-16T10:00:40,513 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=75a5a39ee7bf008864556f27870615b9, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T10:00:40,513 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=75a5a39ee7bf008864556f27870615b9, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T10:00:40,513 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37267 {}] ipc.CallRunner(138): callId: 111 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34428 deadline: 1731751300513, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=75a5a39ee7bf008864556f27870615b9, server=037c58a19ddb,37267,1731751077844 2024-11-16T10:00:40,513 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] ipc.CallRunner(138): callId: 118 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34430 deadline: 1731751300513, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=75a5a39ee7bf008864556f27870615b9, server=037c58a19ddb,37267,1731751077844 2024-11-16T10:00:40,513 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731751217321.75a5a39ee7bf008864556f27870615b9., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 , the old value is region=TestAcidGuarantees,,1731751217321.75a5a39ee7bf008864556f27870615b9., hostname=037c58a19ddb,37267,1731751077844, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=75a5a39ee7bf008864556f27870615b9, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:00:40,513 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731751217321.75a5a39ee7bf008864556f27870615b9., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=75a5a39ee7bf008864556f27870615b9, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:00:40,513 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731751217321.75a5a39ee7bf008864556f27870615b9., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 because the exception is null or not the one we care about 2024-11-16T10:00:40,513 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731751217321.75a5a39ee7bf008864556f27870615b9., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 , the old value is region=TestAcidGuarantees,,1731751217321.75a5a39ee7bf008864556f27870615b9., hostname=037c58a19ddb,37267,1731751077844, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=75a5a39ee7bf008864556f27870615b9, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:00:40,513 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731751217321.75a5a39ee7bf008864556f27870615b9., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 , the old value is region=TestAcidGuarantees,,1731751217321.75a5a39ee7bf008864556f27870615b9., hostname=037c58a19ddb,37267,1731751077844, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=75a5a39ee7bf008864556f27870615b9, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:00:40,513 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731751217321.75a5a39ee7bf008864556f27870615b9., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=75a5a39ee7bf008864556f27870615b9, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:00:40,513 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731751217321.75a5a39ee7bf008864556f27870615b9., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=75a5a39ee7bf008864556f27870615b9, server=037c58a19ddb,37267,1731751077844 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:00:40,513 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731751217321.75a5a39ee7bf008864556f27870615b9., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 because the exception is null or not the one we care about 2024-11-16T10:00:40,513 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731751217321.75a5a39ee7bf008864556f27870615b9., hostname=037c58a19ddb,37267,1731751077844, seqNum=5 because the exception is null or not the one we care about 2024-11-16T10:00:40,578 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=335 (bloomFilter=true), to=hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/75a5a39ee7bf008864556f27870615b9/.tmp/C/c97e7346574145ee8de55ce290fdd612 2024-11-16T10:00:40,581 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/75a5a39ee7bf008864556f27870615b9/.tmp/A/b5de0c395eb44babbf679ef96e17b28b as hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/75a5a39ee7bf008864556f27870615b9/A/b5de0c395eb44babbf679ef96e17b28b 2024-11-16T10:00:40,583 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/75a5a39ee7bf008864556f27870615b9/A/b5de0c395eb44babbf679ef96e17b28b, entries=250, sequenceid=335, filesize=47.5 K 2024-11-16T10:00:40,584 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/75a5a39ee7bf008864556f27870615b9/.tmp/B/c7fd2d10579b4772ab5af7d1b90dd669 as hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/75a5a39ee7bf008864556f27870615b9/B/c7fd2d10579b4772ab5af7d1b90dd669 2024-11-16T10:00:40,586 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/75a5a39ee7bf008864556f27870615b9/B/c7fd2d10579b4772ab5af7d1b90dd669, entries=150, sequenceid=335, filesize=12.0 K 2024-11-16T10:00:40,587 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/75a5a39ee7bf008864556f27870615b9/.tmp/C/c97e7346574145ee8de55ce290fdd612 as hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/75a5a39ee7bf008864556f27870615b9/C/c97e7346574145ee8de55ce290fdd612 2024-11-16T10:00:40,589 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/75a5a39ee7bf008864556f27870615b9/C/c97e7346574145ee8de55ce290fdd612, entries=150, sequenceid=335, filesize=12.0 K 2024-11-16T10:00:40,590 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~80.51 KB/82440, heapSize ~211.64 KB/216720, currentSize=120.76 KB/123660 for 75a5a39ee7bf008864556f27870615b9 in 1254ms, sequenceid=335, compaction requested=true 2024-11-16T10:00:40,590 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 75a5a39ee7bf008864556f27870615b9: 2024-11-16T10:00:40,590 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 75a5a39ee7bf008864556f27870615b9:A, priority=-2147483648, current under compaction store size is 1 2024-11-16T10:00:40,590 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-16T10:00:40,590 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 75a5a39ee7bf008864556f27870615b9:B, priority=-2147483648, current under compaction store size is 2 2024-11-16T10:00:40,590 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-16T10:00:40,590 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 75a5a39ee7bf008864556f27870615b9:C, priority=-2147483648, current under compaction store size is 3 2024-11-16T10:00:40,590 DEBUG [RS:0;037c58a19ddb:37267-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-16T10:00:40,590 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-16T10:00:40,590 DEBUG [RS:0;037c58a19ddb:37267-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-16T10:00:40,591 DEBUG [RS:0;037c58a19ddb:37267-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 195282 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-16T10:00:40,591 DEBUG [RS:0;037c58a19ddb:37267-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 49852 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-16T10:00:40,591 DEBUG [RS:0;037c58a19ddb:37267-shortCompactions-0 {}] regionserver.HStore(1541): 75a5a39ee7bf008864556f27870615b9/A is initiating minor compaction (all files) 2024-11-16T10:00:40,591 DEBUG [RS:0;037c58a19ddb:37267-longCompactions-0 {}] regionserver.HStore(1541): 75a5a39ee7bf008864556f27870615b9/B is initiating minor compaction (all files) 2024-11-16T10:00:40,591 INFO [RS:0;037c58a19ddb:37267-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 75a5a39ee7bf008864556f27870615b9/B in TestAcidGuarantees,,1731751217321.75a5a39ee7bf008864556f27870615b9. 2024-11-16T10:00:40,591 INFO [RS:0;037c58a19ddb:37267-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 75a5a39ee7bf008864556f27870615b9/A in TestAcidGuarantees,,1731751217321.75a5a39ee7bf008864556f27870615b9. 2024-11-16T10:00:40,591 INFO [RS:0;037c58a19ddb:37267-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/75a5a39ee7bf008864556f27870615b9/A/a9799a08cb1d4d1d974bd50f1f7666ee, hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/75a5a39ee7bf008864556f27870615b9/A/c22cfe4cb7594389be246f1bb4cf7a91, hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/75a5a39ee7bf008864556f27870615b9/A/28b43766060b4a8aa1c7479ce4e97c1a, hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/75a5a39ee7bf008864556f27870615b9/A/b5de0c395eb44babbf679ef96e17b28b] into tmpdir=hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/75a5a39ee7bf008864556f27870615b9/.tmp, totalSize=190.7 K 2024-11-16T10:00:40,591 INFO [RS:0;037c58a19ddb:37267-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/75a5a39ee7bf008864556f27870615b9/B/bcb295d9c71e4707b33a2662fd1e76e0, hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/75a5a39ee7bf008864556f27870615b9/B/43a66c20363745609f16e83a6a762ca7, hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/75a5a39ee7bf008864556f27870615b9/B/a6494e0ab0dd42e7885c245e52425bfe, hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/75a5a39ee7bf008864556f27870615b9/B/c7fd2d10579b4772ab5af7d1b90dd669] into tmpdir=hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/75a5a39ee7bf008864556f27870615b9/.tmp, totalSize=48.7 K 2024-11-16T10:00:40,591 INFO [RS:0;037c58a19ddb:37267-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=12 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731751217321.75a5a39ee7bf008864556f27870615b9. 2024-11-16T10:00:40,591 DEBUG [RS:0;037c58a19ddb:37267-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731751217321.75a5a39ee7bf008864556f27870615b9. files: [hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/75a5a39ee7bf008864556f27870615b9/A/a9799a08cb1d4d1d974bd50f1f7666ee, hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/75a5a39ee7bf008864556f27870615b9/A/c22cfe4cb7594389be246f1bb4cf7a91, hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/75a5a39ee7bf008864556f27870615b9/A/28b43766060b4a8aa1c7479ce4e97c1a, hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/75a5a39ee7bf008864556f27870615b9/A/b5de0c395eb44babbf679ef96e17b28b] 2024-11-16T10:00:40,591 DEBUG [RS:0;037c58a19ddb:37267-longCompactions-0 {}] compactions.Compactor(225): Compacting bcb295d9c71e4707b33a2662fd1e76e0, keycount=150, bloomtype=ROW, size=12.6 K, encoding=NONE, compression=NONE, seqNum=281, earliestPutTs=1731751235344 2024-11-16T10:00:40,591 DEBUG [RS:0;037c58a19ddb:37267-shortCompactions-0 {}] compactions.Compactor(225): Compacting a9799a08cb1d4d1d974bd50f1f7666ee, keycount=150, bloomtype=ROW, size=31.2 K, encoding=NONE, compression=NONE, seqNum=281, earliestPutTs=1731751235344 2024-11-16T10:00:40,591 DEBUG [RS:0;037c58a19ddb:37267-longCompactions-0 {}] compactions.Compactor(225): Compacting 43a66c20363745609f16e83a6a762ca7, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=296, earliestPutTs=1731751236508 2024-11-16T10:00:40,591 DEBUG [RS:0;037c58a19ddb:37267-shortCompactions-0 {}] compactions.Compactor(225): Compacting c22cfe4cb7594389be246f1bb4cf7a91, keycount=450, bloomtype=ROW, size=81.5 K, encoding=NONE, compression=NONE, seqNum=296, earliestPutTs=1731751235347 2024-11-16T10:00:40,591 DEBUG [RS:0;037c58a19ddb:37267-longCompactions-0 {}] compactions.Compactor(225): Compacting a6494e0ab0dd42e7885c245e52425bfe, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=320, earliestPutTs=1731751236538 2024-11-16T10:00:40,592 DEBUG [RS:0;037c58a19ddb:37267-shortCompactions-0 {}] compactions.Compactor(225): Compacting 28b43766060b4a8aa1c7479ce4e97c1a, keycount=150, bloomtype=ROW, size=30.5 K, encoding=NONE, compression=NONE, seqNum=320, earliestPutTs=1731751236538 2024-11-16T10:00:40,592 DEBUG [RS:0;037c58a19ddb:37267-longCompactions-0 {}] compactions.Compactor(225): Compacting c7fd2d10579b4772ab5af7d1b90dd669, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=335, earliestPutTs=1731751238700 2024-11-16T10:00:40,592 DEBUG [RS:0;037c58a19ddb:37267-shortCompactions-0 {}] compactions.Compactor(225): Compacting b5de0c395eb44babbf679ef96e17b28b, keycount=250, bloomtype=ROW, size=47.5 K, encoding=NONE, compression=NONE, seqNum=335, earliestPutTs=1731751238700 2024-11-16T10:00:40,597 INFO [RS:0;037c58a19ddb:37267-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=75a5a39ee7bf008864556f27870615b9] 2024-11-16T10:00:40,598 DEBUG [RS:0;037c58a19ddb:37267-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241116f116c56f452447c9b94393f5939f74eb_75a5a39ee7bf008864556f27870615b9 store=[table=TestAcidGuarantees family=A region=75a5a39ee7bf008864556f27870615b9] 2024-11-16T10:00:40,599 INFO [RS:0;037c58a19ddb:37267-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 75a5a39ee7bf008864556f27870615b9#B#compaction#536 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-16T10:00:40,599 DEBUG [RS:0;037c58a19ddb:37267-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/75a5a39ee7bf008864556f27870615b9/.tmp/B/ebd24934665b43f791337c23491de394 is 50, key is test_row_0/B:col10/1731751239335/Put/seqid=0 2024-11-16T10:00:40,603 DEBUG [RS:0;037c58a19ddb:37267-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241116f116c56f452447c9b94393f5939f74eb_75a5a39ee7bf008864556f27870615b9, store=[table=TestAcidGuarantees family=A region=75a5a39ee7bf008864556f27870615b9] 2024-11-16T10:00:40,603 DEBUG [RS:0;037c58a19ddb:37267-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241116f116c56f452447c9b94393f5939f74eb_75a5a39ee7bf008864556f27870615b9 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=75a5a39ee7bf008864556f27870615b9] 2024-11-16T10:00:40,604 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37591 is added to blk_1073742459_1635 (size=13085) 2024-11-16T10:00:40,615 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37591 is added to blk_1073742460_1636 (size=4469) 2024-11-16T10:00:40,616 INFO [RS:0;037c58a19ddb:37267-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 75a5a39ee7bf008864556f27870615b9#A#compaction#535 average throughput is 1.29 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-16T10:00:40,617 DEBUG [RS:0;037c58a19ddb:37267-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/75a5a39ee7bf008864556f27870615b9/.tmp/A/aeeec544c6e045fd98f6f043d922bb2c is 175, key is test_row_0/A:col10/1731751239335/Put/seqid=0 2024-11-16T10:00:40,619 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37591 is added to blk_1073742461_1637 (size=32039) 2024-11-16T10:00:40,623 DEBUG [RS:0;037c58a19ddb:37267-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/75a5a39ee7bf008864556f27870615b9/.tmp/A/aeeec544c6e045fd98f6f043d922bb2c as hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/75a5a39ee7bf008864556f27870615b9/A/aeeec544c6e045fd98f6f043d922bb2c 2024-11-16T10:00:40,626 INFO [RS:0;037c58a19ddb:37267-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 75a5a39ee7bf008864556f27870615b9/A of 75a5a39ee7bf008864556f27870615b9 into aeeec544c6e045fd98f6f043d922bb2c(size=31.3 K), total size for store is 31.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-16T10:00:40,626 DEBUG [RS:0;037c58a19ddb:37267-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 75a5a39ee7bf008864556f27870615b9: 2024-11-16T10:00:40,626 INFO [RS:0;037c58a19ddb:37267-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731751217321.75a5a39ee7bf008864556f27870615b9., storeName=75a5a39ee7bf008864556f27870615b9/A, priority=12, startTime=1731751240590; duration=0sec 2024-11-16T10:00:40,627 DEBUG [RS:0;037c58a19ddb:37267-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-16T10:00:40,627 DEBUG [RS:0;037c58a19ddb:37267-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 75a5a39ee7bf008864556f27870615b9:A 2024-11-16T10:00:40,627 DEBUG [RS:0;037c58a19ddb:37267-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-16T10:00:40,628 DEBUG [RS:0;037c58a19ddb:37267-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 49852 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-16T10:00:40,628 DEBUG [RS:0;037c58a19ddb:37267-shortCompactions-0 {}] regionserver.HStore(1541): 75a5a39ee7bf008864556f27870615b9/C is initiating minor compaction (all files) 2024-11-16T10:00:40,628 INFO [RS:0;037c58a19ddb:37267-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 75a5a39ee7bf008864556f27870615b9/C in TestAcidGuarantees,,1731751217321.75a5a39ee7bf008864556f27870615b9. 2024-11-16T10:00:40,628 INFO [RS:0;037c58a19ddb:37267-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/75a5a39ee7bf008864556f27870615b9/C/3c4ca7c586824ed4a1983ce0fef4addd, hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/75a5a39ee7bf008864556f27870615b9/C/d05779c165b8492fa69eef9f745d8ec7, hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/75a5a39ee7bf008864556f27870615b9/C/de57cafc4e144f88a43a19eb66c92da3, hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/75a5a39ee7bf008864556f27870615b9/C/c97e7346574145ee8de55ce290fdd612] into tmpdir=hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/75a5a39ee7bf008864556f27870615b9/.tmp, totalSize=48.7 K 2024-11-16T10:00:40,628 DEBUG [RS:0;037c58a19ddb:37267-shortCompactions-0 {}] compactions.Compactor(225): Compacting 3c4ca7c586824ed4a1983ce0fef4addd, keycount=150, bloomtype=ROW, size=12.6 K, encoding=NONE, compression=NONE, seqNum=281, earliestPutTs=1731751235344 2024-11-16T10:00:40,628 DEBUG [RS:0;037c58a19ddb:37267-shortCompactions-0 {}] compactions.Compactor(225): Compacting d05779c165b8492fa69eef9f745d8ec7, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=296, earliestPutTs=1731751236508 2024-11-16T10:00:40,628 DEBUG [RS:0;037c58a19ddb:37267-shortCompactions-0 {}] compactions.Compactor(225): Compacting de57cafc4e144f88a43a19eb66c92da3, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=320, earliestPutTs=1731751236538 2024-11-16T10:00:40,629 DEBUG [RS:0;037c58a19ddb:37267-shortCompactions-0 {}] compactions.Compactor(225): Compacting c97e7346574145ee8de55ce290fdd612, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=335, earliestPutTs=1731751238700 2024-11-16T10:00:40,635 INFO [RS:0;037c58a19ddb:37267-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 75a5a39ee7bf008864556f27870615b9#C#compaction#537 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-16T10:00:40,635 DEBUG [RS:0;037c58a19ddb:37267-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/75a5a39ee7bf008864556f27870615b9/.tmp/C/7699d0174b814f91b24e3e26bef66b00 is 50, key is test_row_0/C:col10/1731751239335/Put/seqid=0 2024-11-16T10:00:40,638 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37591 is added to blk_1073742462_1638 (size=13085) 2024-11-16T10:00:40,703 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37877 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=162 2024-11-16T10:00:40,703 INFO [RPCClient-NioEventLoopGroup-4-12 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-16T10:00:41,008 DEBUG [RS:0;037c58a19ddb:37267-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/75a5a39ee7bf008864556f27870615b9/.tmp/B/ebd24934665b43f791337c23491de394 as hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/75a5a39ee7bf008864556f27870615b9/B/ebd24934665b43f791337c23491de394 2024-11-16T10:00:41,012 INFO [RS:0;037c58a19ddb:37267-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 75a5a39ee7bf008864556f27870615b9/B of 75a5a39ee7bf008864556f27870615b9 into ebd24934665b43f791337c23491de394(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-16T10:00:41,012 DEBUG [RS:0;037c58a19ddb:37267-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 75a5a39ee7bf008864556f27870615b9: 2024-11-16T10:00:41,012 INFO [RS:0;037c58a19ddb:37267-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731751217321.75a5a39ee7bf008864556f27870615b9., storeName=75a5a39ee7bf008864556f27870615b9/B, priority=12, startTime=1731751240590; duration=0sec 2024-11-16T10:00:41,012 DEBUG [RS:0;037c58a19ddb:37267-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-16T10:00:41,012 DEBUG [RS:0;037c58a19ddb:37267-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 75a5a39ee7bf008864556f27870615b9:B 2024-11-16T10:00:41,042 DEBUG [RS:0;037c58a19ddb:37267-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/75a5a39ee7bf008864556f27870615b9/.tmp/C/7699d0174b814f91b24e3e26bef66b00 as hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/75a5a39ee7bf008864556f27870615b9/C/7699d0174b814f91b24e3e26bef66b00 2024-11-16T10:00:41,045 INFO [RS:0;037c58a19ddb:37267-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 75a5a39ee7bf008864556f27870615b9/C of 75a5a39ee7bf008864556f27870615b9 into 7699d0174b814f91b24e3e26bef66b00(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-16T10:00:41,045 DEBUG [RS:0;037c58a19ddb:37267-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 75a5a39ee7bf008864556f27870615b9: 2024-11-16T10:00:41,045 INFO [RS:0;037c58a19ddb:37267-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731751217321.75a5a39ee7bf008864556f27870615b9., storeName=75a5a39ee7bf008864556f27870615b9/C, priority=12, startTime=1731751240590; duration=0sec 2024-11-16T10:00:41,045 DEBUG [RS:0;037c58a19ddb:37267-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-16T10:00:41,045 DEBUG [RS:0;037c58a19ddb:37267-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 75a5a39ee7bf008864556f27870615b9:C 2024-11-16T10:00:41,514 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37267 {}] regionserver.HRegion(8855): Flush requested on 75a5a39ee7bf008864556f27870615b9 2024-11-16T10:00:41,514 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-16T10:00:41,515 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 75a5a39ee7bf008864556f27870615b9 3/3 column families, dataSize=127.47 KB heapSize=334.73 KB 2024-11-16T10:00:41,515 INFO [Thread-2414 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2414. 2024-11-16T10:00:41,515 DEBUG [Thread-2414 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-16T10:00:41,515 DEBUG [Thread-2414 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T10:00:41,515 DEBUG [Thread-2414 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T10:00:41,515 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-16T10:00:41,520 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241116c36ecbbc687c46e5b29621a2aef88260_75a5a39ee7bf008864556f27870615b9 is 50, key is test_row_0/A:col10/1731751239347/Put/seqid=0 2024-11-16T10:00:41,522 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37591 is added to blk_1073742463_1639 (size=12454) 2024-11-16T10:00:41,524 INFO [Thread-2422 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2422. 2024-11-16T10:00:41,524 DEBUG [Thread-2422 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-16T10:00:41,524 DEBUG [Thread-2422 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T10:00:41,524 DEBUG [Thread-2422 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T10:00:41,524 INFO [Thread-2416 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2416. 2024-11-16T10:00:41,524 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-16T10:00:41,524 DEBUG [Thread-2416 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-16T10:00:41,524 INFO [Thread-2418 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2418. 2024-11-16T10:00:41,524 DEBUG [Thread-2416 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T10:00:41,524 DEBUG [Thread-2418 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-16T10:00:41,524 DEBUG [Thread-2416 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T10:00:41,524 DEBUG [Thread-2418 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T10:00:41,524 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-16T10:00:41,524 DEBUG [Thread-2418 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T10:00:41,524 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-16T10:00:41,923 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:41,926 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241116c36ecbbc687c46e5b29621a2aef88260_75a5a39ee7bf008864556f27870615b9 to hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241116c36ecbbc687c46e5b29621a2aef88260_75a5a39ee7bf008864556f27870615b9 2024-11-16T10:00:41,927 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/75a5a39ee7bf008864556f27870615b9/.tmp/A/14cf530290ee418eb52b46e872342c90, store: [table=TestAcidGuarantees family=A region=75a5a39ee7bf008864556f27870615b9] 2024-11-16T10:00:41,927 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/75a5a39ee7bf008864556f27870615b9/.tmp/A/14cf530290ee418eb52b46e872342c90 is 175, key is test_row_0/A:col10/1731751239347/Put/seqid=0 2024-11-16T10:00:41,930 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37591 is added to blk_1073742464_1640 (size=31255) 2024-11-16T10:00:42,331 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=360, memsize=42.5 K, hasBloomFilter=true, into tmp file hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/75a5a39ee7bf008864556f27870615b9/.tmp/A/14cf530290ee418eb52b46e872342c90 2024-11-16T10:00:42,336 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/75a5a39ee7bf008864556f27870615b9/.tmp/B/7df72da7667d414a9d7338ddc300f9e8 is 50, key is test_row_0/B:col10/1731751239347/Put/seqid=0 2024-11-16T10:00:42,340 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37591 is added to blk_1073742465_1641 (size=12301) 2024-11-16T10:00:42,740 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=360 (bloomFilter=true), to=hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/75a5a39ee7bf008864556f27870615b9/.tmp/B/7df72da7667d414a9d7338ddc300f9e8 2024-11-16T10:00:42,745 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/75a5a39ee7bf008864556f27870615b9/.tmp/C/d38616733ef944879101f438e711e5ee is 50, key is test_row_0/C:col10/1731751239347/Put/seqid=0 2024-11-16T10:00:42,749 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37591 is added to blk_1073742466_1642 (size=12301) 2024-11-16T10:00:42,964 INFO [Thread-2420 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2420. 2024-11-16T10:00:42,964 DEBUG [Thread-2420 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-16T10:00:42,964 DEBUG [Thread-2420 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T10:00:42,964 DEBUG [Thread-2420 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T10:00:42,964 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-16T10:00:42,965 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(392): Finished test. Writers: 2024-11-16T10:00:42,965 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 59 2024-11-16T10:00:42,965 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 60 2024-11-16T10:00:42,965 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 61 2024-11-16T10:00:42,965 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 51 2024-11-16T10:00:42,965 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 54 2024-11-16T10:00:42,965 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(396): Readers: 2024-11-16T10:00:42,965 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 3732 2024-11-16T10:00:42,965 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 3812 2024-11-16T10:00:42,965 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 3861 2024-11-16T10:00:42,965 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 3737 2024-11-16T10:00:42,965 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 3816 2024-11-16T10:00:42,965 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(400): Scanners: 2024-11-16T10:00:42,965 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-16T10:00:42,965 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool.doWork(AcidGuaranteesTestTool.java:131) at org.apache.hadoop.hbase.util.AbstractHBaseTool.run(AbstractHBaseTool.java:150) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.runTestAtomicity(AcidGuaranteesTestBase.java:102) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.testMobGetAtomicity(AcidGuaranteesTestBase.java:122) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-16T10:00:42,965 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T10:00:42,965 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T10:00:42,965 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-16T10:00:42,966 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37877 {}] master.HMaster$13(2820): Client=jenkins//172.17.0.2 disable TestAcidGuarantees 2024-11-16T10:00:42,967 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37877 {}] procedure2.ProcedureExecutor(1139): Stored pid=164, state=RUNNABLE:DISABLE_TABLE_PREPARE, hasLock=false; DisableTableProcedure table=TestAcidGuarantees 2024-11-16T10:00:42,968 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37877 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=164 2024-11-16T10:00:42,968 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731751242968"}]},"ts":"1731751242968"} 2024-11-16T10:00:42,969 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLING in hbase:meta 2024-11-16T10:00:42,969 INFO [PEWorker-2 {}] procedure.DisableTableProcedure(284): Set TestAcidGuarantees to state=DISABLING 2024-11-16T10:00:42,970 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=165, ppid=164, state=RUNNABLE:CLOSE_TABLE_REGIONS_SCHEDULE, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees}] 2024-11-16T10:00:42,971 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=166, ppid=165, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=75a5a39ee7bf008864556f27870615b9, UNASSIGN}] 2024-11-16T10:00:42,971 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=166, ppid=165, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=75a5a39ee7bf008864556f27870615b9, UNASSIGN 2024-11-16T10:00:42,972 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=166 updating hbase:meta row=75a5a39ee7bf008864556f27870615b9, regionState=CLOSING, regionLocation=037c58a19ddb,37267,1731751077844 2024-11-16T10:00:42,973 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=166, ppid=165, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=75a5a39ee7bf008864556f27870615b9, UNASSIGN because future has completed 2024-11-16T10:00:42,973 DEBUG [PEWorker-3 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-11-16T10:00:42,973 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=167, ppid=166, state=RUNNABLE, hasLock=false; CloseRegionProcedure 75a5a39ee7bf008864556f27870615b9, server=037c58a19ddb,37267,1731751077844}] 2024-11-16T10:00:43,073 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37877 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=164 2024-11-16T10:00:43,125 INFO [RS_CLOSE_REGION-regionserver/037c58a19ddb:0-0 {event_type=M_RS_CLOSE_REGION, pid=167}] handler.UnassignRegionHandler(122): Close 75a5a39ee7bf008864556f27870615b9 2024-11-16T10:00:43,125 DEBUG [RS_CLOSE_REGION-regionserver/037c58a19ddb:0-0 {event_type=M_RS_CLOSE_REGION, pid=167}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-11-16T10:00:43,125 DEBUG [RS_CLOSE_REGION-regionserver/037c58a19ddb:0-0 {event_type=M_RS_CLOSE_REGION, pid=167}] regionserver.HRegion(1722): Closing 75a5a39ee7bf008864556f27870615b9, disabling compactions & flushes 2024-11-16T10:00:43,125 DEBUG [RS_CLOSE_REGION-regionserver/037c58a19ddb:0-0 {event_type=M_RS_CLOSE_REGION, pid=167}] regionserver.HRegion(1993): waiting for 0 compactions & cache flush to complete for region TestAcidGuarantees,,1731751217321.75a5a39ee7bf008864556f27870615b9. 2024-11-16T10:00:43,149 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=360 (bloomFilter=true), to=hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/75a5a39ee7bf008864556f27870615b9/.tmp/C/d38616733ef944879101f438e711e5ee 2024-11-16T10:00:43,152 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/75a5a39ee7bf008864556f27870615b9/.tmp/A/14cf530290ee418eb52b46e872342c90 as hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/75a5a39ee7bf008864556f27870615b9/A/14cf530290ee418eb52b46e872342c90 2024-11-16T10:00:43,155 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/75a5a39ee7bf008864556f27870615b9/A/14cf530290ee418eb52b46e872342c90, entries=150, sequenceid=360, filesize=30.5 K 2024-11-16T10:00:43,155 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/75a5a39ee7bf008864556f27870615b9/.tmp/B/7df72da7667d414a9d7338ddc300f9e8 as hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/75a5a39ee7bf008864556f27870615b9/B/7df72da7667d414a9d7338ddc300f9e8 2024-11-16T10:00:43,158 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/75a5a39ee7bf008864556f27870615b9/B/7df72da7667d414a9d7338ddc300f9e8, entries=150, sequenceid=360, filesize=12.0 K 2024-11-16T10:00:43,158 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/75a5a39ee7bf008864556f27870615b9/.tmp/C/d38616733ef944879101f438e711e5ee as hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/75a5a39ee7bf008864556f27870615b9/C/d38616733ef944879101f438e711e5ee 2024-11-16T10:00:43,161 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/75a5a39ee7bf008864556f27870615b9/C/d38616733ef944879101f438e711e5ee, entries=150, sequenceid=360, filesize=12.0 K 2024-11-16T10:00:43,161 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~127.47 KB/130530, heapSize ~334.69 KB/342720, currentSize=26.84 KB/27480 for 75a5a39ee7bf008864556f27870615b9 in 1647ms, sequenceid=360, compaction requested=false 2024-11-16T10:00:43,161 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 75a5a39ee7bf008864556f27870615b9: 2024-11-16T10:00:43,161 INFO [RS_CLOSE_REGION-regionserver/037c58a19ddb:0-0 {event_type=M_RS_CLOSE_REGION, pid=167}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1731751217321.75a5a39ee7bf008864556f27870615b9. 2024-11-16T10:00:43,161 DEBUG [RS_CLOSE_REGION-regionserver/037c58a19ddb:0-0 {event_type=M_RS_CLOSE_REGION, pid=167}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1731751217321.75a5a39ee7bf008864556f27870615b9. 2024-11-16T10:00:43,161 DEBUG [RS_CLOSE_REGION-regionserver/037c58a19ddb:0-0 {event_type=M_RS_CLOSE_REGION, pid=167}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1731751217321.75a5a39ee7bf008864556f27870615b9. after waiting 0 ms 2024-11-16T10:00:43,161 DEBUG [RS_CLOSE_REGION-regionserver/037c58a19ddb:0-0 {event_type=M_RS_CLOSE_REGION, pid=167}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1731751217321.75a5a39ee7bf008864556f27870615b9. 2024-11-16T10:00:43,161 INFO [RS_CLOSE_REGION-regionserver/037c58a19ddb:0-0 {event_type=M_RS_CLOSE_REGION, pid=167}] regionserver.HRegion(2902): Flushing 75a5a39ee7bf008864556f27870615b9 3/3 column families, dataSize=26.84 KB heapSize=71.06 KB 2024-11-16T10:00:43,165 DEBUG [RS_CLOSE_REGION-regionserver/037c58a19ddb:0-0 {event_type=M_RS_CLOSE_REGION, pid=167}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411160485287b50a24cd88e73f77fcf346e0e_75a5a39ee7bf008864556f27870615b9 is 50, key is test_row_1/A:col10/1731751242963/Put/seqid=0 2024-11-16T10:00:43,168 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37591 is added to blk_1073742467_1643 (size=9914) 2024-11-16T10:00:43,283 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37877 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=164 2024-11-16T10:00:43,569 DEBUG [RS_CLOSE_REGION-regionserver/037c58a19ddb:0-0 {event_type=M_RS_CLOSE_REGION, pid=167}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:00:43,571 INFO [RS_CLOSE_REGION-regionserver/037c58a19ddb:0-0 {event_type=M_RS_CLOSE_REGION, pid=167}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411160485287b50a24cd88e73f77fcf346e0e_75a5a39ee7bf008864556f27870615b9 to hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411160485287b50a24cd88e73f77fcf346e0e_75a5a39ee7bf008864556f27870615b9 2024-11-16T10:00:43,572 DEBUG [RS_CLOSE_REGION-regionserver/037c58a19ddb:0-0 {event_type=M_RS_CLOSE_REGION, pid=167}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/75a5a39ee7bf008864556f27870615b9/.tmp/A/1a7cfac30c36489c915f34000108b971, store: [table=TestAcidGuarantees family=A region=75a5a39ee7bf008864556f27870615b9] 2024-11-16T10:00:43,572 DEBUG [RS_CLOSE_REGION-regionserver/037c58a19ddb:0-0 {event_type=M_RS_CLOSE_REGION, pid=167}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/75a5a39ee7bf008864556f27870615b9/.tmp/A/1a7cfac30c36489c915f34000108b971 is 175, key is test_row_1/A:col10/1731751242963/Put/seqid=0 2024-11-16T10:00:43,575 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37591 is added to blk_1073742468_1644 (size=22561) 2024-11-16T10:00:43,575 INFO [RS_CLOSE_REGION-regionserver/037c58a19ddb:0-0 {event_type=M_RS_CLOSE_REGION, pid=167}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=367, memsize=8.9 K, hasBloomFilter=true, into tmp file hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/75a5a39ee7bf008864556f27870615b9/.tmp/A/1a7cfac30c36489c915f34000108b971 2024-11-16T10:00:43,580 DEBUG [RS_CLOSE_REGION-regionserver/037c58a19ddb:0-0 {event_type=M_RS_CLOSE_REGION, pid=167}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/75a5a39ee7bf008864556f27870615b9/.tmp/B/4d2d6037543d47ed9866526a5500da6e is 50, key is test_row_1/B:col10/1731751242963/Put/seqid=0 2024-11-16T10:00:43,583 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37591 is added to blk_1073742469_1645 (size=9857) 2024-11-16T10:00:43,592 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37877 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=164 2024-11-16T10:00:43,983 INFO [RS_CLOSE_REGION-regionserver/037c58a19ddb:0-0 {event_type=M_RS_CLOSE_REGION, pid=167}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=8.95 KB at sequenceid=367 (bloomFilter=true), to=hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/75a5a39ee7bf008864556f27870615b9/.tmp/B/4d2d6037543d47ed9866526a5500da6e 2024-11-16T10:00:43,988 DEBUG [RS_CLOSE_REGION-regionserver/037c58a19ddb:0-0 {event_type=M_RS_CLOSE_REGION, pid=167}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/75a5a39ee7bf008864556f27870615b9/.tmp/C/be34b726ba824dbbac877913c92dfcc4 is 50, key is test_row_1/C:col10/1731751242963/Put/seqid=0 2024-11-16T10:00:43,991 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37591 is added to blk_1073742470_1646 (size=9857) 2024-11-16T10:00:44,103 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37877 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=164 2024-11-16T10:00:44,392 INFO [RS_CLOSE_REGION-regionserver/037c58a19ddb:0-0 {event_type=M_RS_CLOSE_REGION, pid=167}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=8.95 KB at sequenceid=367 (bloomFilter=true), to=hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/75a5a39ee7bf008864556f27870615b9/.tmp/C/be34b726ba824dbbac877913c92dfcc4 2024-11-16T10:00:44,395 DEBUG [RS_CLOSE_REGION-regionserver/037c58a19ddb:0-0 {event_type=M_RS_CLOSE_REGION, pid=167}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/75a5a39ee7bf008864556f27870615b9/.tmp/A/1a7cfac30c36489c915f34000108b971 as hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/75a5a39ee7bf008864556f27870615b9/A/1a7cfac30c36489c915f34000108b971 2024-11-16T10:00:44,398 INFO [RS_CLOSE_REGION-regionserver/037c58a19ddb:0-0 {event_type=M_RS_CLOSE_REGION, pid=167}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/75a5a39ee7bf008864556f27870615b9/A/1a7cfac30c36489c915f34000108b971, entries=100, sequenceid=367, filesize=22.0 K 2024-11-16T10:00:44,398 DEBUG [RS_CLOSE_REGION-regionserver/037c58a19ddb:0-0 {event_type=M_RS_CLOSE_REGION, pid=167}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/75a5a39ee7bf008864556f27870615b9/.tmp/B/4d2d6037543d47ed9866526a5500da6e as hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/75a5a39ee7bf008864556f27870615b9/B/4d2d6037543d47ed9866526a5500da6e 2024-11-16T10:00:44,401 INFO [RS_CLOSE_REGION-regionserver/037c58a19ddb:0-0 {event_type=M_RS_CLOSE_REGION, pid=167}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/75a5a39ee7bf008864556f27870615b9/B/4d2d6037543d47ed9866526a5500da6e, entries=100, sequenceid=367, filesize=9.6 K 2024-11-16T10:00:44,401 DEBUG [RS_CLOSE_REGION-regionserver/037c58a19ddb:0-0 {event_type=M_RS_CLOSE_REGION, pid=167}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/75a5a39ee7bf008864556f27870615b9/.tmp/C/be34b726ba824dbbac877913c92dfcc4 as hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/75a5a39ee7bf008864556f27870615b9/C/be34b726ba824dbbac877913c92dfcc4 2024-11-16T10:00:44,404 INFO [RS_CLOSE_REGION-regionserver/037c58a19ddb:0-0 {event_type=M_RS_CLOSE_REGION, pid=167}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/75a5a39ee7bf008864556f27870615b9/C/be34b726ba824dbbac877913c92dfcc4, entries=100, sequenceid=367, filesize=9.6 K 2024-11-16T10:00:44,404 INFO [RS_CLOSE_REGION-regionserver/037c58a19ddb:0-0 {event_type=M_RS_CLOSE_REGION, pid=167}] regionserver.HRegion(3140): Finished flush of dataSize ~26.84 KB/27480, heapSize ~71.02 KB/72720, currentSize=0 B/0 for 75a5a39ee7bf008864556f27870615b9 in 1243ms, sequenceid=367, compaction requested=true 2024-11-16T10:00:44,405 DEBUG [StoreCloser-TestAcidGuarantees,,1731751217321.75a5a39ee7bf008864556f27870615b9.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/75a5a39ee7bf008864556f27870615b9/A/a08f489d2f324f8c94bc70f75b31e4c0, hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/75a5a39ee7bf008864556f27870615b9/A/892f7393aaa849a5a4ba84fc0f0c0c2b, hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/75a5a39ee7bf008864556f27870615b9/A/6bc1ee565b014b8782a1e180b4df00da, hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/75a5a39ee7bf008864556f27870615b9/A/56efa3c087bc425a838c68ca08f65887, hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/75a5a39ee7bf008864556f27870615b9/A/9712b27b0e4f495e9650ab7a025b0ba1, hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/75a5a39ee7bf008864556f27870615b9/A/7559ef990f3f4cefab9b80a49e667063, hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/75a5a39ee7bf008864556f27870615b9/A/c45414a6fed343d78f84a9867a4a9bfc, hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/75a5a39ee7bf008864556f27870615b9/A/a229e16c29e64ca982c262db7c64505c, hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/75a5a39ee7bf008864556f27870615b9/A/6b95983c3e394d52bd279882723afe46, hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/75a5a39ee7bf008864556f27870615b9/A/2567f08bfa9a407898aa25582d9a0dff, hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/75a5a39ee7bf008864556f27870615b9/A/80e1e8f18c254519b6be286bcb29f3db, hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/75a5a39ee7bf008864556f27870615b9/A/5ba81c6f8c6544a29da2e79b2c8b49d0, hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/75a5a39ee7bf008864556f27870615b9/A/955ac682fed54cc0af1fa49e6c502dd2, hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/75a5a39ee7bf008864556f27870615b9/A/b9bdffdf7c5d49568463afe11efd7753, hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/75a5a39ee7bf008864556f27870615b9/A/584e3cb61e0b4e15acdabe73ca47b987, hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/75a5a39ee7bf008864556f27870615b9/A/f234c94f31c345bdacdbdad119738a8c, hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/75a5a39ee7bf008864556f27870615b9/A/698550d7a97e47fe8a23c3fa8834d663, hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/75a5a39ee7bf008864556f27870615b9/A/a15a55fc49b74b35997765299aa047dc, hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/75a5a39ee7bf008864556f27870615b9/A/889382ab4777414f88c6fe4df51e78eb, hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/75a5a39ee7bf008864556f27870615b9/A/a9799a08cb1d4d1d974bd50f1f7666ee, hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/75a5a39ee7bf008864556f27870615b9/A/c22cfe4cb7594389be246f1bb4cf7a91, hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/75a5a39ee7bf008864556f27870615b9/A/28b43766060b4a8aa1c7479ce4e97c1a, hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/75a5a39ee7bf008864556f27870615b9/A/b5de0c395eb44babbf679ef96e17b28b] to archive 2024-11-16T10:00:44,406 DEBUG [StoreCloser-TestAcidGuarantees,,1731751217321.75a5a39ee7bf008864556f27870615b9.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-16T10:00:44,407 DEBUG [StoreCloser-TestAcidGuarantees,,1731751217321.75a5a39ee7bf008864556f27870615b9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/75a5a39ee7bf008864556f27870615b9/A/a08f489d2f324f8c94bc70f75b31e4c0 to hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/archive/data/default/TestAcidGuarantees/75a5a39ee7bf008864556f27870615b9/A/a08f489d2f324f8c94bc70f75b31e4c0 2024-11-16T10:00:44,407 DEBUG [StoreCloser-TestAcidGuarantees,,1731751217321.75a5a39ee7bf008864556f27870615b9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/75a5a39ee7bf008864556f27870615b9/A/892f7393aaa849a5a4ba84fc0f0c0c2b to hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/archive/data/default/TestAcidGuarantees/75a5a39ee7bf008864556f27870615b9/A/892f7393aaa849a5a4ba84fc0f0c0c2b 2024-11-16T10:00:44,408 DEBUG [StoreCloser-TestAcidGuarantees,,1731751217321.75a5a39ee7bf008864556f27870615b9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/75a5a39ee7bf008864556f27870615b9/A/6bc1ee565b014b8782a1e180b4df00da to hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/archive/data/default/TestAcidGuarantees/75a5a39ee7bf008864556f27870615b9/A/6bc1ee565b014b8782a1e180b4df00da 2024-11-16T10:00:44,409 DEBUG [StoreCloser-TestAcidGuarantees,,1731751217321.75a5a39ee7bf008864556f27870615b9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/75a5a39ee7bf008864556f27870615b9/A/56efa3c087bc425a838c68ca08f65887 to hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/archive/data/default/TestAcidGuarantees/75a5a39ee7bf008864556f27870615b9/A/56efa3c087bc425a838c68ca08f65887 2024-11-16T10:00:44,410 DEBUG [StoreCloser-TestAcidGuarantees,,1731751217321.75a5a39ee7bf008864556f27870615b9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/75a5a39ee7bf008864556f27870615b9/A/9712b27b0e4f495e9650ab7a025b0ba1 to hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/archive/data/default/TestAcidGuarantees/75a5a39ee7bf008864556f27870615b9/A/9712b27b0e4f495e9650ab7a025b0ba1 2024-11-16T10:00:44,411 DEBUG [StoreCloser-TestAcidGuarantees,,1731751217321.75a5a39ee7bf008864556f27870615b9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/75a5a39ee7bf008864556f27870615b9/A/7559ef990f3f4cefab9b80a49e667063 to hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/archive/data/default/TestAcidGuarantees/75a5a39ee7bf008864556f27870615b9/A/7559ef990f3f4cefab9b80a49e667063 2024-11-16T10:00:44,412 DEBUG [StoreCloser-TestAcidGuarantees,,1731751217321.75a5a39ee7bf008864556f27870615b9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/75a5a39ee7bf008864556f27870615b9/A/c45414a6fed343d78f84a9867a4a9bfc to hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/archive/data/default/TestAcidGuarantees/75a5a39ee7bf008864556f27870615b9/A/c45414a6fed343d78f84a9867a4a9bfc 2024-11-16T10:00:44,412 DEBUG [StoreCloser-TestAcidGuarantees,,1731751217321.75a5a39ee7bf008864556f27870615b9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/75a5a39ee7bf008864556f27870615b9/A/a229e16c29e64ca982c262db7c64505c to hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/archive/data/default/TestAcidGuarantees/75a5a39ee7bf008864556f27870615b9/A/a229e16c29e64ca982c262db7c64505c 2024-11-16T10:00:44,413 DEBUG [StoreCloser-TestAcidGuarantees,,1731751217321.75a5a39ee7bf008864556f27870615b9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/75a5a39ee7bf008864556f27870615b9/A/6b95983c3e394d52bd279882723afe46 to hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/archive/data/default/TestAcidGuarantees/75a5a39ee7bf008864556f27870615b9/A/6b95983c3e394d52bd279882723afe46 2024-11-16T10:00:44,414 DEBUG [StoreCloser-TestAcidGuarantees,,1731751217321.75a5a39ee7bf008864556f27870615b9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/75a5a39ee7bf008864556f27870615b9/A/2567f08bfa9a407898aa25582d9a0dff to hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/archive/data/default/TestAcidGuarantees/75a5a39ee7bf008864556f27870615b9/A/2567f08bfa9a407898aa25582d9a0dff 2024-11-16T10:00:44,414 DEBUG [StoreCloser-TestAcidGuarantees,,1731751217321.75a5a39ee7bf008864556f27870615b9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/75a5a39ee7bf008864556f27870615b9/A/80e1e8f18c254519b6be286bcb29f3db to hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/archive/data/default/TestAcidGuarantees/75a5a39ee7bf008864556f27870615b9/A/80e1e8f18c254519b6be286bcb29f3db 2024-11-16T10:00:44,415 DEBUG [StoreCloser-TestAcidGuarantees,,1731751217321.75a5a39ee7bf008864556f27870615b9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/75a5a39ee7bf008864556f27870615b9/A/5ba81c6f8c6544a29da2e79b2c8b49d0 to hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/archive/data/default/TestAcidGuarantees/75a5a39ee7bf008864556f27870615b9/A/5ba81c6f8c6544a29da2e79b2c8b49d0 2024-11-16T10:00:44,416 DEBUG [StoreCloser-TestAcidGuarantees,,1731751217321.75a5a39ee7bf008864556f27870615b9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/75a5a39ee7bf008864556f27870615b9/A/955ac682fed54cc0af1fa49e6c502dd2 to hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/archive/data/default/TestAcidGuarantees/75a5a39ee7bf008864556f27870615b9/A/955ac682fed54cc0af1fa49e6c502dd2 2024-11-16T10:00:44,417 DEBUG [StoreCloser-TestAcidGuarantees,,1731751217321.75a5a39ee7bf008864556f27870615b9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/75a5a39ee7bf008864556f27870615b9/A/b9bdffdf7c5d49568463afe11efd7753 to hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/archive/data/default/TestAcidGuarantees/75a5a39ee7bf008864556f27870615b9/A/b9bdffdf7c5d49568463afe11efd7753 2024-11-16T10:00:44,417 DEBUG [StoreCloser-TestAcidGuarantees,,1731751217321.75a5a39ee7bf008864556f27870615b9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/75a5a39ee7bf008864556f27870615b9/A/584e3cb61e0b4e15acdabe73ca47b987 to hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/archive/data/default/TestAcidGuarantees/75a5a39ee7bf008864556f27870615b9/A/584e3cb61e0b4e15acdabe73ca47b987 2024-11-16T10:00:44,418 DEBUG [StoreCloser-TestAcidGuarantees,,1731751217321.75a5a39ee7bf008864556f27870615b9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/75a5a39ee7bf008864556f27870615b9/A/f234c94f31c345bdacdbdad119738a8c to hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/archive/data/default/TestAcidGuarantees/75a5a39ee7bf008864556f27870615b9/A/f234c94f31c345bdacdbdad119738a8c 2024-11-16T10:00:44,419 DEBUG [StoreCloser-TestAcidGuarantees,,1731751217321.75a5a39ee7bf008864556f27870615b9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/75a5a39ee7bf008864556f27870615b9/A/698550d7a97e47fe8a23c3fa8834d663 to hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/archive/data/default/TestAcidGuarantees/75a5a39ee7bf008864556f27870615b9/A/698550d7a97e47fe8a23c3fa8834d663 2024-11-16T10:00:44,420 DEBUG [StoreCloser-TestAcidGuarantees,,1731751217321.75a5a39ee7bf008864556f27870615b9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/75a5a39ee7bf008864556f27870615b9/A/a15a55fc49b74b35997765299aa047dc to hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/archive/data/default/TestAcidGuarantees/75a5a39ee7bf008864556f27870615b9/A/a15a55fc49b74b35997765299aa047dc 2024-11-16T10:00:44,421 DEBUG [StoreCloser-TestAcidGuarantees,,1731751217321.75a5a39ee7bf008864556f27870615b9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/75a5a39ee7bf008864556f27870615b9/A/889382ab4777414f88c6fe4df51e78eb to hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/archive/data/default/TestAcidGuarantees/75a5a39ee7bf008864556f27870615b9/A/889382ab4777414f88c6fe4df51e78eb 2024-11-16T10:00:44,421 DEBUG [StoreCloser-TestAcidGuarantees,,1731751217321.75a5a39ee7bf008864556f27870615b9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/75a5a39ee7bf008864556f27870615b9/A/a9799a08cb1d4d1d974bd50f1f7666ee to hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/archive/data/default/TestAcidGuarantees/75a5a39ee7bf008864556f27870615b9/A/a9799a08cb1d4d1d974bd50f1f7666ee 2024-11-16T10:00:44,422 DEBUG [StoreCloser-TestAcidGuarantees,,1731751217321.75a5a39ee7bf008864556f27870615b9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/75a5a39ee7bf008864556f27870615b9/A/c22cfe4cb7594389be246f1bb4cf7a91 to hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/archive/data/default/TestAcidGuarantees/75a5a39ee7bf008864556f27870615b9/A/c22cfe4cb7594389be246f1bb4cf7a91 2024-11-16T10:00:44,423 DEBUG [StoreCloser-TestAcidGuarantees,,1731751217321.75a5a39ee7bf008864556f27870615b9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/75a5a39ee7bf008864556f27870615b9/A/28b43766060b4a8aa1c7479ce4e97c1a to hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/archive/data/default/TestAcidGuarantees/75a5a39ee7bf008864556f27870615b9/A/28b43766060b4a8aa1c7479ce4e97c1a 2024-11-16T10:00:44,423 DEBUG [StoreCloser-TestAcidGuarantees,,1731751217321.75a5a39ee7bf008864556f27870615b9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/75a5a39ee7bf008864556f27870615b9/A/b5de0c395eb44babbf679ef96e17b28b to hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/archive/data/default/TestAcidGuarantees/75a5a39ee7bf008864556f27870615b9/A/b5de0c395eb44babbf679ef96e17b28b 2024-11-16T10:00:44,425 DEBUG [StoreCloser-TestAcidGuarantees,,1731751217321.75a5a39ee7bf008864556f27870615b9.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/75a5a39ee7bf008864556f27870615b9/B/932ed69b7379451ea7d13e6008653448, hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/75a5a39ee7bf008864556f27870615b9/B/134c29f339e34f62805c458513d26ee8, hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/75a5a39ee7bf008864556f27870615b9/B/2a05f259717c4e27b74a376fd6d40bf9, hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/75a5a39ee7bf008864556f27870615b9/B/bd56f650318741fcbc136f933fd4496c, hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/75a5a39ee7bf008864556f27870615b9/B/3807f7330657475a8b56368c81de0fe8, hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/75a5a39ee7bf008864556f27870615b9/B/371f5fd3a9984c5a9022b16ad2d69107, hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/75a5a39ee7bf008864556f27870615b9/B/34673bb2b5ce43d1825af91265a180ad, hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/75a5a39ee7bf008864556f27870615b9/B/8c92dd9bd0d34f40b1dab0876669e75e, hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/75a5a39ee7bf008864556f27870615b9/B/53b261f643444e2f9eeda0200e77f83c, hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/75a5a39ee7bf008864556f27870615b9/B/a1291712189d4c8882e6915360928d4f, hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/75a5a39ee7bf008864556f27870615b9/B/36b2600aa11f4987924ad638f9bf96a5, hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/75a5a39ee7bf008864556f27870615b9/B/0cef038cd17c43eab448fd42a00414aa, hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/75a5a39ee7bf008864556f27870615b9/B/d4e00a75365b4c6e937dcf823b92da7c, hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/75a5a39ee7bf008864556f27870615b9/B/eb93c6fdfe03475481ef8e2ff93de934, hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/75a5a39ee7bf008864556f27870615b9/B/fdace4a2305248da8f37008895cb2774, hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/75a5a39ee7bf008864556f27870615b9/B/2dac6b2aba484c8ca3588f74e5a36789, hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/75a5a39ee7bf008864556f27870615b9/B/518123bb8da8418cb9cb610105faac82, hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/75a5a39ee7bf008864556f27870615b9/B/e59fe4cb81fc403999f5bb2b146958fc, hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/75a5a39ee7bf008864556f27870615b9/B/bcb295d9c71e4707b33a2662fd1e76e0, hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/75a5a39ee7bf008864556f27870615b9/B/4c0a41e3bba648e79161d541f37c4939, hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/75a5a39ee7bf008864556f27870615b9/B/43a66c20363745609f16e83a6a762ca7, hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/75a5a39ee7bf008864556f27870615b9/B/a6494e0ab0dd42e7885c245e52425bfe, hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/75a5a39ee7bf008864556f27870615b9/B/c7fd2d10579b4772ab5af7d1b90dd669] to archive 2024-11-16T10:00:44,425 DEBUG [StoreCloser-TestAcidGuarantees,,1731751217321.75a5a39ee7bf008864556f27870615b9.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-16T10:00:44,426 DEBUG [StoreCloser-TestAcidGuarantees,,1731751217321.75a5a39ee7bf008864556f27870615b9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/75a5a39ee7bf008864556f27870615b9/B/932ed69b7379451ea7d13e6008653448 to hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/archive/data/default/TestAcidGuarantees/75a5a39ee7bf008864556f27870615b9/B/932ed69b7379451ea7d13e6008653448 2024-11-16T10:00:44,427 DEBUG [StoreCloser-TestAcidGuarantees,,1731751217321.75a5a39ee7bf008864556f27870615b9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/75a5a39ee7bf008864556f27870615b9/B/134c29f339e34f62805c458513d26ee8 to hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/archive/data/default/TestAcidGuarantees/75a5a39ee7bf008864556f27870615b9/B/134c29f339e34f62805c458513d26ee8 2024-11-16T10:00:44,428 DEBUG [StoreCloser-TestAcidGuarantees,,1731751217321.75a5a39ee7bf008864556f27870615b9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/75a5a39ee7bf008864556f27870615b9/B/2a05f259717c4e27b74a376fd6d40bf9 to hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/archive/data/default/TestAcidGuarantees/75a5a39ee7bf008864556f27870615b9/B/2a05f259717c4e27b74a376fd6d40bf9 2024-11-16T10:00:44,428 DEBUG [StoreCloser-TestAcidGuarantees,,1731751217321.75a5a39ee7bf008864556f27870615b9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/75a5a39ee7bf008864556f27870615b9/B/bd56f650318741fcbc136f933fd4496c to hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/archive/data/default/TestAcidGuarantees/75a5a39ee7bf008864556f27870615b9/B/bd56f650318741fcbc136f933fd4496c 2024-11-16T10:00:44,429 DEBUG [StoreCloser-TestAcidGuarantees,,1731751217321.75a5a39ee7bf008864556f27870615b9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/75a5a39ee7bf008864556f27870615b9/B/3807f7330657475a8b56368c81de0fe8 to hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/archive/data/default/TestAcidGuarantees/75a5a39ee7bf008864556f27870615b9/B/3807f7330657475a8b56368c81de0fe8 2024-11-16T10:00:44,430 DEBUG [StoreCloser-TestAcidGuarantees,,1731751217321.75a5a39ee7bf008864556f27870615b9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/75a5a39ee7bf008864556f27870615b9/B/371f5fd3a9984c5a9022b16ad2d69107 to hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/archive/data/default/TestAcidGuarantees/75a5a39ee7bf008864556f27870615b9/B/371f5fd3a9984c5a9022b16ad2d69107 2024-11-16T10:00:44,430 DEBUG [StoreCloser-TestAcidGuarantees,,1731751217321.75a5a39ee7bf008864556f27870615b9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/75a5a39ee7bf008864556f27870615b9/B/34673bb2b5ce43d1825af91265a180ad to hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/archive/data/default/TestAcidGuarantees/75a5a39ee7bf008864556f27870615b9/B/34673bb2b5ce43d1825af91265a180ad 2024-11-16T10:00:44,431 DEBUG [StoreCloser-TestAcidGuarantees,,1731751217321.75a5a39ee7bf008864556f27870615b9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/75a5a39ee7bf008864556f27870615b9/B/8c92dd9bd0d34f40b1dab0876669e75e to hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/archive/data/default/TestAcidGuarantees/75a5a39ee7bf008864556f27870615b9/B/8c92dd9bd0d34f40b1dab0876669e75e 2024-11-16T10:00:44,432 DEBUG [StoreCloser-TestAcidGuarantees,,1731751217321.75a5a39ee7bf008864556f27870615b9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/75a5a39ee7bf008864556f27870615b9/B/53b261f643444e2f9eeda0200e77f83c to hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/archive/data/default/TestAcidGuarantees/75a5a39ee7bf008864556f27870615b9/B/53b261f643444e2f9eeda0200e77f83c 2024-11-16T10:00:44,433 DEBUG [StoreCloser-TestAcidGuarantees,,1731751217321.75a5a39ee7bf008864556f27870615b9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/75a5a39ee7bf008864556f27870615b9/B/a1291712189d4c8882e6915360928d4f to hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/archive/data/default/TestAcidGuarantees/75a5a39ee7bf008864556f27870615b9/B/a1291712189d4c8882e6915360928d4f 2024-11-16T10:00:44,433 DEBUG [StoreCloser-TestAcidGuarantees,,1731751217321.75a5a39ee7bf008864556f27870615b9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/75a5a39ee7bf008864556f27870615b9/B/36b2600aa11f4987924ad638f9bf96a5 to hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/archive/data/default/TestAcidGuarantees/75a5a39ee7bf008864556f27870615b9/B/36b2600aa11f4987924ad638f9bf96a5 2024-11-16T10:00:44,434 DEBUG [StoreCloser-TestAcidGuarantees,,1731751217321.75a5a39ee7bf008864556f27870615b9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/75a5a39ee7bf008864556f27870615b9/B/0cef038cd17c43eab448fd42a00414aa to hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/archive/data/default/TestAcidGuarantees/75a5a39ee7bf008864556f27870615b9/B/0cef038cd17c43eab448fd42a00414aa 2024-11-16T10:00:44,435 DEBUG [StoreCloser-TestAcidGuarantees,,1731751217321.75a5a39ee7bf008864556f27870615b9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/75a5a39ee7bf008864556f27870615b9/B/d4e00a75365b4c6e937dcf823b92da7c to hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/archive/data/default/TestAcidGuarantees/75a5a39ee7bf008864556f27870615b9/B/d4e00a75365b4c6e937dcf823b92da7c 2024-11-16T10:00:44,435 DEBUG [StoreCloser-TestAcidGuarantees,,1731751217321.75a5a39ee7bf008864556f27870615b9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/75a5a39ee7bf008864556f27870615b9/B/eb93c6fdfe03475481ef8e2ff93de934 to hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/archive/data/default/TestAcidGuarantees/75a5a39ee7bf008864556f27870615b9/B/eb93c6fdfe03475481ef8e2ff93de934 2024-11-16T10:00:44,436 DEBUG [StoreCloser-TestAcidGuarantees,,1731751217321.75a5a39ee7bf008864556f27870615b9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/75a5a39ee7bf008864556f27870615b9/B/fdace4a2305248da8f37008895cb2774 to hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/archive/data/default/TestAcidGuarantees/75a5a39ee7bf008864556f27870615b9/B/fdace4a2305248da8f37008895cb2774 2024-11-16T10:00:44,437 DEBUG [StoreCloser-TestAcidGuarantees,,1731751217321.75a5a39ee7bf008864556f27870615b9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/75a5a39ee7bf008864556f27870615b9/B/2dac6b2aba484c8ca3588f74e5a36789 to hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/archive/data/default/TestAcidGuarantees/75a5a39ee7bf008864556f27870615b9/B/2dac6b2aba484c8ca3588f74e5a36789 2024-11-16T10:00:44,438 DEBUG [StoreCloser-TestAcidGuarantees,,1731751217321.75a5a39ee7bf008864556f27870615b9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/75a5a39ee7bf008864556f27870615b9/B/518123bb8da8418cb9cb610105faac82 to hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/archive/data/default/TestAcidGuarantees/75a5a39ee7bf008864556f27870615b9/B/518123bb8da8418cb9cb610105faac82 2024-11-16T10:00:44,438 DEBUG [StoreCloser-TestAcidGuarantees,,1731751217321.75a5a39ee7bf008864556f27870615b9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/75a5a39ee7bf008864556f27870615b9/B/e59fe4cb81fc403999f5bb2b146958fc to hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/archive/data/default/TestAcidGuarantees/75a5a39ee7bf008864556f27870615b9/B/e59fe4cb81fc403999f5bb2b146958fc 2024-11-16T10:00:44,439 DEBUG [StoreCloser-TestAcidGuarantees,,1731751217321.75a5a39ee7bf008864556f27870615b9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/75a5a39ee7bf008864556f27870615b9/B/bcb295d9c71e4707b33a2662fd1e76e0 to hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/archive/data/default/TestAcidGuarantees/75a5a39ee7bf008864556f27870615b9/B/bcb295d9c71e4707b33a2662fd1e76e0 2024-11-16T10:00:44,440 DEBUG [StoreCloser-TestAcidGuarantees,,1731751217321.75a5a39ee7bf008864556f27870615b9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/75a5a39ee7bf008864556f27870615b9/B/4c0a41e3bba648e79161d541f37c4939 to hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/archive/data/default/TestAcidGuarantees/75a5a39ee7bf008864556f27870615b9/B/4c0a41e3bba648e79161d541f37c4939 2024-11-16T10:00:44,441 DEBUG [StoreCloser-TestAcidGuarantees,,1731751217321.75a5a39ee7bf008864556f27870615b9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/75a5a39ee7bf008864556f27870615b9/B/43a66c20363745609f16e83a6a762ca7 to hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/archive/data/default/TestAcidGuarantees/75a5a39ee7bf008864556f27870615b9/B/43a66c20363745609f16e83a6a762ca7 2024-11-16T10:00:44,441 DEBUG [StoreCloser-TestAcidGuarantees,,1731751217321.75a5a39ee7bf008864556f27870615b9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/75a5a39ee7bf008864556f27870615b9/B/a6494e0ab0dd42e7885c245e52425bfe to hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/archive/data/default/TestAcidGuarantees/75a5a39ee7bf008864556f27870615b9/B/a6494e0ab0dd42e7885c245e52425bfe 2024-11-16T10:00:44,442 DEBUG [StoreCloser-TestAcidGuarantees,,1731751217321.75a5a39ee7bf008864556f27870615b9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/75a5a39ee7bf008864556f27870615b9/B/c7fd2d10579b4772ab5af7d1b90dd669 to hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/archive/data/default/TestAcidGuarantees/75a5a39ee7bf008864556f27870615b9/B/c7fd2d10579b4772ab5af7d1b90dd669 2024-11-16T10:00:44,443 DEBUG [StoreCloser-TestAcidGuarantees,,1731751217321.75a5a39ee7bf008864556f27870615b9.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/75a5a39ee7bf008864556f27870615b9/C/ba3e707513614fbe871b02ce8b163935, hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/75a5a39ee7bf008864556f27870615b9/C/ccab6fd1079046d3b9bd608ce597d14a, hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/75a5a39ee7bf008864556f27870615b9/C/660a6b8d351a4709a39d1668e126c1e3, hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/75a5a39ee7bf008864556f27870615b9/C/ac18b2b07c4b45a3a8d847ee65be1577, hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/75a5a39ee7bf008864556f27870615b9/C/4634faf075a148e0aa55658f0b57d51f, hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/75a5a39ee7bf008864556f27870615b9/C/c36f6d576030452db8db9b4cf83d536e, hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/75a5a39ee7bf008864556f27870615b9/C/d29bb71ca61440e28f0cbc943b187152, hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/75a5a39ee7bf008864556f27870615b9/C/6b0168243cd5427aa60eaa61687c5d65, hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/75a5a39ee7bf008864556f27870615b9/C/22cf205d63134391b67b77b703ef814d, hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/75a5a39ee7bf008864556f27870615b9/C/659904f59b7c4a5aa6ff2a8d9f0a61c3, hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/75a5a39ee7bf008864556f27870615b9/C/38bbc22a1129485f90b4650be6aa0326, hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/75a5a39ee7bf008864556f27870615b9/C/eece82162ad545eea9eedfee8e3109fc, hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/75a5a39ee7bf008864556f27870615b9/C/bb43f99d4a98465cbfa41dff2bac86f9, hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/75a5a39ee7bf008864556f27870615b9/C/9c5660694a7a480caf5c5fc83c189dc4, hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/75a5a39ee7bf008864556f27870615b9/C/39ac7213ec73453dabe0a6d1fbad50a7, hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/75a5a39ee7bf008864556f27870615b9/C/c252c96455d64fc29b7b1cda3f9f2f13, hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/75a5a39ee7bf008864556f27870615b9/C/26e3e62e9e914fa28e8cff1840ba7691, hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/75a5a39ee7bf008864556f27870615b9/C/f158f302c9e4447abd35ad6d46fa5b3e, hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/75a5a39ee7bf008864556f27870615b9/C/3c4ca7c586824ed4a1983ce0fef4addd, hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/75a5a39ee7bf008864556f27870615b9/C/2f41815c61aa4891bf17f02eb08cfa8e, hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/75a5a39ee7bf008864556f27870615b9/C/d05779c165b8492fa69eef9f745d8ec7, hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/75a5a39ee7bf008864556f27870615b9/C/de57cafc4e144f88a43a19eb66c92da3, hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/75a5a39ee7bf008864556f27870615b9/C/c97e7346574145ee8de55ce290fdd612] to archive 2024-11-16T10:00:44,444 DEBUG [StoreCloser-TestAcidGuarantees,,1731751217321.75a5a39ee7bf008864556f27870615b9.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-16T10:00:44,445 DEBUG [StoreCloser-TestAcidGuarantees,,1731751217321.75a5a39ee7bf008864556f27870615b9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/75a5a39ee7bf008864556f27870615b9/C/ba3e707513614fbe871b02ce8b163935 to hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/archive/data/default/TestAcidGuarantees/75a5a39ee7bf008864556f27870615b9/C/ba3e707513614fbe871b02ce8b163935 2024-11-16T10:00:44,446 DEBUG [StoreCloser-TestAcidGuarantees,,1731751217321.75a5a39ee7bf008864556f27870615b9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/75a5a39ee7bf008864556f27870615b9/C/ccab6fd1079046d3b9bd608ce597d14a to hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/archive/data/default/TestAcidGuarantees/75a5a39ee7bf008864556f27870615b9/C/ccab6fd1079046d3b9bd608ce597d14a 2024-11-16T10:00:44,446 DEBUG [StoreCloser-TestAcidGuarantees,,1731751217321.75a5a39ee7bf008864556f27870615b9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/75a5a39ee7bf008864556f27870615b9/C/660a6b8d351a4709a39d1668e126c1e3 to hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/archive/data/default/TestAcidGuarantees/75a5a39ee7bf008864556f27870615b9/C/660a6b8d351a4709a39d1668e126c1e3 2024-11-16T10:00:44,447 DEBUG [StoreCloser-TestAcidGuarantees,,1731751217321.75a5a39ee7bf008864556f27870615b9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/75a5a39ee7bf008864556f27870615b9/C/ac18b2b07c4b45a3a8d847ee65be1577 to hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/archive/data/default/TestAcidGuarantees/75a5a39ee7bf008864556f27870615b9/C/ac18b2b07c4b45a3a8d847ee65be1577 2024-11-16T10:00:44,448 DEBUG [StoreCloser-TestAcidGuarantees,,1731751217321.75a5a39ee7bf008864556f27870615b9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/75a5a39ee7bf008864556f27870615b9/C/4634faf075a148e0aa55658f0b57d51f to hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/archive/data/default/TestAcidGuarantees/75a5a39ee7bf008864556f27870615b9/C/4634faf075a148e0aa55658f0b57d51f 2024-11-16T10:00:44,449 DEBUG [StoreCloser-TestAcidGuarantees,,1731751217321.75a5a39ee7bf008864556f27870615b9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/75a5a39ee7bf008864556f27870615b9/C/c36f6d576030452db8db9b4cf83d536e to hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/archive/data/default/TestAcidGuarantees/75a5a39ee7bf008864556f27870615b9/C/c36f6d576030452db8db9b4cf83d536e 2024-11-16T10:00:44,449 DEBUG [StoreCloser-TestAcidGuarantees,,1731751217321.75a5a39ee7bf008864556f27870615b9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/75a5a39ee7bf008864556f27870615b9/C/d29bb71ca61440e28f0cbc943b187152 to hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/archive/data/default/TestAcidGuarantees/75a5a39ee7bf008864556f27870615b9/C/d29bb71ca61440e28f0cbc943b187152 2024-11-16T10:00:44,450 DEBUG [StoreCloser-TestAcidGuarantees,,1731751217321.75a5a39ee7bf008864556f27870615b9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/75a5a39ee7bf008864556f27870615b9/C/6b0168243cd5427aa60eaa61687c5d65 to hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/archive/data/default/TestAcidGuarantees/75a5a39ee7bf008864556f27870615b9/C/6b0168243cd5427aa60eaa61687c5d65 2024-11-16T10:00:44,451 DEBUG [StoreCloser-TestAcidGuarantees,,1731751217321.75a5a39ee7bf008864556f27870615b9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/75a5a39ee7bf008864556f27870615b9/C/22cf205d63134391b67b77b703ef814d to hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/archive/data/default/TestAcidGuarantees/75a5a39ee7bf008864556f27870615b9/C/22cf205d63134391b67b77b703ef814d 2024-11-16T10:00:44,452 DEBUG [StoreCloser-TestAcidGuarantees,,1731751217321.75a5a39ee7bf008864556f27870615b9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/75a5a39ee7bf008864556f27870615b9/C/659904f59b7c4a5aa6ff2a8d9f0a61c3 to hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/archive/data/default/TestAcidGuarantees/75a5a39ee7bf008864556f27870615b9/C/659904f59b7c4a5aa6ff2a8d9f0a61c3 2024-11-16T10:00:44,453 DEBUG [StoreCloser-TestAcidGuarantees,,1731751217321.75a5a39ee7bf008864556f27870615b9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/75a5a39ee7bf008864556f27870615b9/C/38bbc22a1129485f90b4650be6aa0326 to hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/archive/data/default/TestAcidGuarantees/75a5a39ee7bf008864556f27870615b9/C/38bbc22a1129485f90b4650be6aa0326 2024-11-16T10:00:44,453 DEBUG [StoreCloser-TestAcidGuarantees,,1731751217321.75a5a39ee7bf008864556f27870615b9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/75a5a39ee7bf008864556f27870615b9/C/eece82162ad545eea9eedfee8e3109fc to hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/archive/data/default/TestAcidGuarantees/75a5a39ee7bf008864556f27870615b9/C/eece82162ad545eea9eedfee8e3109fc 2024-11-16T10:00:44,454 DEBUG [StoreCloser-TestAcidGuarantees,,1731751217321.75a5a39ee7bf008864556f27870615b9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/75a5a39ee7bf008864556f27870615b9/C/bb43f99d4a98465cbfa41dff2bac86f9 to hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/archive/data/default/TestAcidGuarantees/75a5a39ee7bf008864556f27870615b9/C/bb43f99d4a98465cbfa41dff2bac86f9 2024-11-16T10:00:44,455 DEBUG [StoreCloser-TestAcidGuarantees,,1731751217321.75a5a39ee7bf008864556f27870615b9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/75a5a39ee7bf008864556f27870615b9/C/9c5660694a7a480caf5c5fc83c189dc4 to hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/archive/data/default/TestAcidGuarantees/75a5a39ee7bf008864556f27870615b9/C/9c5660694a7a480caf5c5fc83c189dc4 2024-11-16T10:00:44,456 DEBUG [StoreCloser-TestAcidGuarantees,,1731751217321.75a5a39ee7bf008864556f27870615b9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/75a5a39ee7bf008864556f27870615b9/C/39ac7213ec73453dabe0a6d1fbad50a7 to hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/archive/data/default/TestAcidGuarantees/75a5a39ee7bf008864556f27870615b9/C/39ac7213ec73453dabe0a6d1fbad50a7 2024-11-16T10:00:44,456 DEBUG [StoreCloser-TestAcidGuarantees,,1731751217321.75a5a39ee7bf008864556f27870615b9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/75a5a39ee7bf008864556f27870615b9/C/c252c96455d64fc29b7b1cda3f9f2f13 to hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/archive/data/default/TestAcidGuarantees/75a5a39ee7bf008864556f27870615b9/C/c252c96455d64fc29b7b1cda3f9f2f13 2024-11-16T10:00:44,457 DEBUG [StoreCloser-TestAcidGuarantees,,1731751217321.75a5a39ee7bf008864556f27870615b9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/75a5a39ee7bf008864556f27870615b9/C/26e3e62e9e914fa28e8cff1840ba7691 to hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/archive/data/default/TestAcidGuarantees/75a5a39ee7bf008864556f27870615b9/C/26e3e62e9e914fa28e8cff1840ba7691 2024-11-16T10:00:44,458 DEBUG [StoreCloser-TestAcidGuarantees,,1731751217321.75a5a39ee7bf008864556f27870615b9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/75a5a39ee7bf008864556f27870615b9/C/f158f302c9e4447abd35ad6d46fa5b3e to hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/archive/data/default/TestAcidGuarantees/75a5a39ee7bf008864556f27870615b9/C/f158f302c9e4447abd35ad6d46fa5b3e 2024-11-16T10:00:44,459 DEBUG [StoreCloser-TestAcidGuarantees,,1731751217321.75a5a39ee7bf008864556f27870615b9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/75a5a39ee7bf008864556f27870615b9/C/3c4ca7c586824ed4a1983ce0fef4addd to hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/archive/data/default/TestAcidGuarantees/75a5a39ee7bf008864556f27870615b9/C/3c4ca7c586824ed4a1983ce0fef4addd 2024-11-16T10:00:44,460 DEBUG [StoreCloser-TestAcidGuarantees,,1731751217321.75a5a39ee7bf008864556f27870615b9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/75a5a39ee7bf008864556f27870615b9/C/2f41815c61aa4891bf17f02eb08cfa8e to hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/archive/data/default/TestAcidGuarantees/75a5a39ee7bf008864556f27870615b9/C/2f41815c61aa4891bf17f02eb08cfa8e 2024-11-16T10:00:44,460 DEBUG [StoreCloser-TestAcidGuarantees,,1731751217321.75a5a39ee7bf008864556f27870615b9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/75a5a39ee7bf008864556f27870615b9/C/d05779c165b8492fa69eef9f745d8ec7 to hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/archive/data/default/TestAcidGuarantees/75a5a39ee7bf008864556f27870615b9/C/d05779c165b8492fa69eef9f745d8ec7 2024-11-16T10:00:44,461 DEBUG [StoreCloser-TestAcidGuarantees,,1731751217321.75a5a39ee7bf008864556f27870615b9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/75a5a39ee7bf008864556f27870615b9/C/de57cafc4e144f88a43a19eb66c92da3 to hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/archive/data/default/TestAcidGuarantees/75a5a39ee7bf008864556f27870615b9/C/de57cafc4e144f88a43a19eb66c92da3 2024-11-16T10:00:44,462 DEBUG [StoreCloser-TestAcidGuarantees,,1731751217321.75a5a39ee7bf008864556f27870615b9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/75a5a39ee7bf008864556f27870615b9/C/c97e7346574145ee8de55ce290fdd612 to hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/archive/data/default/TestAcidGuarantees/75a5a39ee7bf008864556f27870615b9/C/c97e7346574145ee8de55ce290fdd612 2024-11-16T10:00:44,465 DEBUG [RS_CLOSE_REGION-regionserver/037c58a19ddb:0-0 {event_type=M_RS_CLOSE_REGION, pid=167}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/75a5a39ee7bf008864556f27870615b9/recovered.edits/370.seqid, newMaxSeqId=370, maxSeqId=4 2024-11-16T10:00:44,466 INFO [RS_CLOSE_REGION-regionserver/037c58a19ddb:0-0 {event_type=M_RS_CLOSE_REGION, pid=167}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1731751217321.75a5a39ee7bf008864556f27870615b9. 2024-11-16T10:00:44,466 DEBUG [RS_CLOSE_REGION-regionserver/037c58a19ddb:0-0 {event_type=M_RS_CLOSE_REGION, pid=167}] regionserver.HRegion(1676): Region close journal for 75a5a39ee7bf008864556f27870615b9: Waiting for close lock at 1731751243125Running coprocessor pre-close hooks at 1731751243125Disabling compacts and flushes for region at 1731751243125Disabling writes for close at 1731751243161 (+36 ms)Obtaining lock to block concurrent updates at 1731751243161Preparing flush snapshotting stores in 75a5a39ee7bf008864556f27870615b9 at 1731751243161Finished memstore snapshotting TestAcidGuarantees,,1731751217321.75a5a39ee7bf008864556f27870615b9., syncing WAL and waiting on mvcc, flushsize=dataSize=27480, getHeapSize=72720, getOffHeapSize=0, getCellsCount=600 at 1731751243162 (+1 ms)Flushing stores of TestAcidGuarantees,,1731751217321.75a5a39ee7bf008864556f27870615b9. at 1731751243162Flushing 75a5a39ee7bf008864556f27870615b9/A: creating writer at 1731751243162Flushing mob file 75a5a39ee7bf008864556f27870615b9/A: appending metadata at 1731751243165 (+3 ms)Flushing mob file 75a5a39ee7bf008864556f27870615b9/A: closing flushed file at 1731751243165Flushing 75a5a39ee7bf008864556f27870615b9/A: appending metadata at 1731751243572 (+407 ms)Flushing 75a5a39ee7bf008864556f27870615b9/A: closing flushed file at 1731751243572Flushing 75a5a39ee7bf008864556f27870615b9/B: creating writer at 1731751243578 (+6 ms)Flushing 75a5a39ee7bf008864556f27870615b9/B: appending metadata at 1731751243580 (+2 ms)Flushing 75a5a39ee7bf008864556f27870615b9/B: closing flushed file at 1731751243580Flushing 75a5a39ee7bf008864556f27870615b9/C: creating writer at 1731751243986 (+406 ms)Flushing 75a5a39ee7bf008864556f27870615b9/C: appending metadata at 1731751243988 (+2 ms)Flushing 75a5a39ee7bf008864556f27870615b9/C: closing flushed file at 1731751243988Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@48cc2959: reopening flushed file at 1731751244395 (+407 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@5a94b635: reopening flushed file at 1731751244398 (+3 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@7c59c39b: reopening flushed file at 1731751244401 (+3 ms)Finished flush of dataSize ~26.84 KB/27480, heapSize ~71.02 KB/72720, currentSize=0 B/0 for 75a5a39ee7bf008864556f27870615b9 in 1243ms, sequenceid=367, compaction requested=true at 1731751244404 (+3 ms)Writing region close event to WAL at 1731751244463 (+59 ms)Running coprocessor post-close hooks at 1731751244466 (+3 ms)Closed at 1731751244466 2024-11-16T10:00:44,467 INFO [RS_CLOSE_REGION-regionserver/037c58a19ddb:0-0 {event_type=M_RS_CLOSE_REGION, pid=167}] handler.UnassignRegionHandler(157): Closed 75a5a39ee7bf008864556f27870615b9 2024-11-16T10:00:44,468 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=166 updating hbase:meta row=75a5a39ee7bf008864556f27870615b9, regionState=CLOSED 2024-11-16T10:00:44,469 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=167, ppid=166, state=RUNNABLE, hasLock=false; CloseRegionProcedure 75a5a39ee7bf008864556f27870615b9, server=037c58a19ddb,37267,1731751077844 because future has completed 2024-11-16T10:00:44,471 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=167, resume processing ppid=166 2024-11-16T10:00:44,472 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=167, ppid=166, state=SUCCESS, hasLock=false; CloseRegionProcedure 75a5a39ee7bf008864556f27870615b9, server=037c58a19ddb,37267,1731751077844 in 1.4970 sec 2024-11-16T10:00:44,472 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=166, resume processing ppid=165 2024-11-16T10:00:44,472 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=166, ppid=165, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=75a5a39ee7bf008864556f27870615b9, UNASSIGN in 1.5000 sec 2024-11-16T10:00:44,474 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=165, resume processing ppid=164 2024-11-16T10:00:44,474 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=165, ppid=164, state=SUCCESS, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees in 1.5030 sec 2024-11-16T10:00:44,475 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731751244475"}]},"ts":"1731751244475"} 2024-11-16T10:00:44,476 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLED in hbase:meta 2024-11-16T10:00:44,476 INFO [PEWorker-5 {}] procedure.DisableTableProcedure(296): Set TestAcidGuarantees to state=DISABLED 2024-11-16T10:00:44,477 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=164, state=SUCCESS, hasLock=false; DisableTableProcedure table=TestAcidGuarantees in 1.5100 sec 2024-11-16T10:00:45,112 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37877 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=164 2024-11-16T10:00:45,113 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DISABLE, Table Name: default:TestAcidGuarantees completed 2024-11-16T10:00:45,113 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37877 {}] master.HMaster$5(2570): Client=jenkins//172.17.0.2 delete TestAcidGuarantees 2024-11-16T10:00:45,114 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37877 {}] procedure2.ProcedureExecutor(1139): Stored pid=168, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees 2024-11-16T10:00:45,115 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(103): Waiting for RIT for pid=168, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-16T10:00:45,116 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37877 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=168 2024-11-16T10:00:45,116 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(115): Deleting regions from filesystem for pid=168, state=RUNNABLE:DELETE_TABLE_CLEAR_FS_LAYOUT, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-16T10:00:45,119 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(131): ARCHIVING hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/75a5a39ee7bf008864556f27870615b9 2024-11-16T10:00:45,121 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(159): Archiving [FileablePath, hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/75a5a39ee7bf008864556f27870615b9/A, FileablePath, hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/75a5a39ee7bf008864556f27870615b9/B, FileablePath, hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/75a5a39ee7bf008864556f27870615b9/C, FileablePath, hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/75a5a39ee7bf008864556f27870615b9/recovered.edits] 2024-11-16T10:00:45,123 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/75a5a39ee7bf008864556f27870615b9/A/14cf530290ee418eb52b46e872342c90 to hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/archive/data/default/TestAcidGuarantees/75a5a39ee7bf008864556f27870615b9/A/14cf530290ee418eb52b46e872342c90 2024-11-16T10:00:45,124 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/75a5a39ee7bf008864556f27870615b9/A/1a7cfac30c36489c915f34000108b971 to hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/archive/data/default/TestAcidGuarantees/75a5a39ee7bf008864556f27870615b9/A/1a7cfac30c36489c915f34000108b971 2024-11-16T10:00:45,125 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/75a5a39ee7bf008864556f27870615b9/A/aeeec544c6e045fd98f6f043d922bb2c to hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/archive/data/default/TestAcidGuarantees/75a5a39ee7bf008864556f27870615b9/A/aeeec544c6e045fd98f6f043d922bb2c 2024-11-16T10:00:45,127 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/75a5a39ee7bf008864556f27870615b9/B/4d2d6037543d47ed9866526a5500da6e to hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/archive/data/default/TestAcidGuarantees/75a5a39ee7bf008864556f27870615b9/B/4d2d6037543d47ed9866526a5500da6e 2024-11-16T10:00:45,128 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/75a5a39ee7bf008864556f27870615b9/B/7df72da7667d414a9d7338ddc300f9e8 to hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/archive/data/default/TestAcidGuarantees/75a5a39ee7bf008864556f27870615b9/B/7df72da7667d414a9d7338ddc300f9e8 2024-11-16T10:00:45,129 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/75a5a39ee7bf008864556f27870615b9/B/ebd24934665b43f791337c23491de394 to hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/archive/data/default/TestAcidGuarantees/75a5a39ee7bf008864556f27870615b9/B/ebd24934665b43f791337c23491de394 2024-11-16T10:00:45,132 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/75a5a39ee7bf008864556f27870615b9/C/7699d0174b814f91b24e3e26bef66b00 to hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/archive/data/default/TestAcidGuarantees/75a5a39ee7bf008864556f27870615b9/C/7699d0174b814f91b24e3e26bef66b00 2024-11-16T10:00:45,133 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/75a5a39ee7bf008864556f27870615b9/C/be34b726ba824dbbac877913c92dfcc4 to hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/archive/data/default/TestAcidGuarantees/75a5a39ee7bf008864556f27870615b9/C/be34b726ba824dbbac877913c92dfcc4 2024-11-16T10:00:45,135 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/75a5a39ee7bf008864556f27870615b9/C/d38616733ef944879101f438e711e5ee to hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/archive/data/default/TestAcidGuarantees/75a5a39ee7bf008864556f27870615b9/C/d38616733ef944879101f438e711e5ee 2024-11-16T10:00:45,137 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/75a5a39ee7bf008864556f27870615b9/recovered.edits/370.seqid to hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/archive/data/default/TestAcidGuarantees/75a5a39ee7bf008864556f27870615b9/recovered.edits/370.seqid 2024-11-16T10:00:45,137 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(610): Deleted hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/default/TestAcidGuarantees/75a5a39ee7bf008864556f27870615b9 2024-11-16T10:00:45,137 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(313): Archived TestAcidGuarantees regions 2024-11-16T10:00:45,138 DEBUG [PEWorker-2 {}] backup.HFileArchiver(131): ARCHIVING hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3 2024-11-16T10:00:45,138 DEBUG [PEWorker-2 {}] backup.HFileArchiver(159): Archiving [FileablePath, hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A] 2024-11-16T10:00:45,140 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411160485287b50a24cd88e73f77fcf346e0e_75a5a39ee7bf008864556f27870615b9 to hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411160485287b50a24cd88e73f77fcf346e0e_75a5a39ee7bf008864556f27870615b9 2024-11-16T10:00:45,141 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111620c9d856242848abb88eb70393008744_75a5a39ee7bf008864556f27870615b9 to hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111620c9d856242848abb88eb70393008744_75a5a39ee7bf008864556f27870615b9 2024-11-16T10:00:45,142 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241116287ae30978a84543903d26b650bf83c2_75a5a39ee7bf008864556f27870615b9 to hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241116287ae30978a84543903d26b650bf83c2_75a5a39ee7bf008864556f27870615b9 2024-11-16T10:00:45,143 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411164c7cb85849044effa54b580940139835_75a5a39ee7bf008864556f27870615b9 to hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411164c7cb85849044effa54b580940139835_75a5a39ee7bf008864556f27870615b9 2024-11-16T10:00:45,144 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111659cbafcb5f8d4b6eb3889e2cda08ed2d_75a5a39ee7bf008864556f27870615b9 to hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111659cbafcb5f8d4b6eb3889e2cda08ed2d_75a5a39ee7bf008864556f27870615b9 2024-11-16T10:00:45,145 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411165f9e28dbe5ae434d9a6b4b81a75e72c8_75a5a39ee7bf008864556f27870615b9 to hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411165f9e28dbe5ae434d9a6b4b81a75e72c8_75a5a39ee7bf008864556f27870615b9 2024-11-16T10:00:45,146 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111663435f957f214c5bb68128e8ac8bcb5c_75a5a39ee7bf008864556f27870615b9 to hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111663435f957f214c5bb68128e8ac8bcb5c_75a5a39ee7bf008864556f27870615b9 2024-11-16T10:00:45,146 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111669b5a8a923f74333a6ac7ef1fa12b14f_75a5a39ee7bf008864556f27870615b9 to hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111669b5a8a923f74333a6ac7ef1fa12b14f_75a5a39ee7bf008864556f27870615b9 2024-11-16T10:00:45,147 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111699da5f2a2dd244dfa3fb0f6d31790c49_75a5a39ee7bf008864556f27870615b9 to hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111699da5f2a2dd244dfa3fb0f6d31790c49_75a5a39ee7bf008864556f27870615b9 2024-11-16T10:00:45,148 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411169de0d3c95d8d4cd69e4c3255e868a303_75a5a39ee7bf008864556f27870615b9 to hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411169de0d3c95d8d4cd69e4c3255e868a303_75a5a39ee7bf008864556f27870615b9 2024-11-16T10:00:45,149 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241116a4d23bce912c47b1accd9d147df295f7_75a5a39ee7bf008864556f27870615b9 to hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241116a4d23bce912c47b1accd9d147df295f7_75a5a39ee7bf008864556f27870615b9 2024-11-16T10:00:45,150 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241116b04b93190a8646dc82267e45f8168d2f_75a5a39ee7bf008864556f27870615b9 to hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241116b04b93190a8646dc82267e45f8168d2f_75a5a39ee7bf008864556f27870615b9 2024-11-16T10:00:45,151 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241116c36ecbbc687c46e5b29621a2aef88260_75a5a39ee7bf008864556f27870615b9 to hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241116c36ecbbc687c46e5b29621a2aef88260_75a5a39ee7bf008864556f27870615b9 2024-11-16T10:00:45,152 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241116cb1ff8adb3814d90a1664ea7e7d28601_75a5a39ee7bf008864556f27870615b9 to hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241116cb1ff8adb3814d90a1664ea7e7d28601_75a5a39ee7bf008864556f27870615b9 2024-11-16T10:00:45,153 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241116db1130e7b0074eb19883b8a8586a467d_75a5a39ee7bf008864556f27870615b9 to hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241116db1130e7b0074eb19883b8a8586a467d_75a5a39ee7bf008864556f27870615b9 2024-11-16T10:00:45,154 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241116dbbb599b2e134d339c1d04eb23bb4b6f_75a5a39ee7bf008864556f27870615b9 to hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241116dbbb599b2e134d339c1d04eb23bb4b6f_75a5a39ee7bf008864556f27870615b9 2024-11-16T10:00:45,155 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241116e29db2783f8f45f1bf1c01e0b8271a4e_75a5a39ee7bf008864556f27870615b9 to hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241116e29db2783f8f45f1bf1c01e0b8271a4e_75a5a39ee7bf008864556f27870615b9 2024-11-16T10:00:45,156 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241116f3686b6e76ea42dbbf0fd2936f4e0b74_75a5a39ee7bf008864556f27870615b9 to hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241116f3686b6e76ea42dbbf0fd2936f4e0b74_75a5a39ee7bf008864556f27870615b9 2024-11-16T10:00:45,156 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241116f7c8313634574e3688e4c0f01e940f78_75a5a39ee7bf008864556f27870615b9 to hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241116f7c8313634574e3688e4c0f01e940f78_75a5a39ee7bf008864556f27870615b9 2024-11-16T10:00:45,157 DEBUG [PEWorker-2 {}] backup.HFileArchiver(610): Deleted hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3 2024-11-16T10:00:45,159 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(120): Deleting regions from META for pid=168, state=RUNNABLE:DELETE_TABLE_REMOVE_FROM_META, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-16T10:00:45,161 WARN [PEWorker-2 {}] procedure.DeleteTableProcedure(370): Deleting some vestigial 1 rows of TestAcidGuarantees from hbase:meta 2024-11-16T10:00:45,162 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(407): Removing 'TestAcidGuarantees' descriptor. 2024-11-16T10:00:45,163 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(126): Deleting assignment state for pid=168, state=RUNNABLE:DELETE_TABLE_UNASSIGN_REGIONS, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-16T10:00:45,163 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(397): Removing 'TestAcidGuarantees' from region states. 2024-11-16T10:00:45,163 DEBUG [PEWorker-2 {}] assignment.RegionStateStore(723): Delete {"totalColumns":1,"row":"TestAcidGuarantees,,1731751217321.75a5a39ee7bf008864556f27870615b9.","families":{"info":[{"qualifier":"","vlen":0,"tag":[],"timestamp":"1731751245163"}]},"ts":"9223372036854775807"} 2024-11-16T10:00:45,165 INFO [PEWorker-2 {}] assignment.RegionStateStore(562): Deleted 1 regions from META 2024-11-16T10:00:45,165 DEBUG [PEWorker-2 {}] assignment.RegionStateStore(563): Deleted regions: [{ENCODED => 75a5a39ee7bf008864556f27870615b9, NAME => 'TestAcidGuarantees,,1731751217321.75a5a39ee7bf008864556f27870615b9.', STARTKEY => '', ENDKEY => ''}] 2024-11-16T10:00:45,165 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(401): Marking 'TestAcidGuarantees' as deleted. 2024-11-16T10:00:45,165 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(964): Delete {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":0,"tag":[],"timestamp":"1731751245165"}]},"ts":"9223372036854775807"} 2024-11-16T10:00:45,166 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(867): Deleted table TestAcidGuarantees state from META 2024-11-16T10:00:45,167 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(133): Finished pid=168, state=RUNNABLE:DELETE_TABLE_POST_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-16T10:00:45,168 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=168, state=SUCCESS, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees in 54 msec 2024-11-16T10:00:45,223 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37877 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=168 2024-11-16T10:00:45,223 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocator(219): Clear meta cache for TestAcidGuarantees 2024-11-16T10:00:45,224 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DELETE, Table Name: default:TestAcidGuarantees completed 2024-11-16T10:00:45,232 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: TestAcidGuaranteesWithNoInMemCompaction#testMobGetAtomicity Thread=242 (was 241) - Thread LEAK? -, OpenFileDescriptor=459 (was 451) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=335 (was 375), ProcessCount=11 (was 11), AvailableMemoryMB=5735 (was 5819) 2024-11-16T10:00:45,232 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1019): Shutting down minicluster 2024-11-16T10:00:45,233 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-16T10:00:45,233 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hbase.thirdparty.com.google.common.io.Closeables.close(Closeables.java:79) at org.apache.hadoop.hbase.HBaseTestingUtil.closeConnection(HBaseTestingUtil.java:2611) at org.apache.hadoop.hbase.HBaseTestingUtil.cleanup(HBaseTestingUtil.java:1065) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniHBaseCluster(HBaseTestingUtil.java:1034) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1020) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.tearDownAfterClass(AcidGuaranteesTestBase.java:66) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-16T10:00:45,233 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T10:00:45,233 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T10:00:45,233 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-16T10:00:45,233 DEBUG [Time-limited test {}] util.JVMClusterUtil(237): Shutting down HBase Cluster 2024-11-16T10:00:45,233 DEBUG [Time-limited test {}] util.JVMClusterUtil(257): Found active master hash=1388394555, stopped=false 2024-11-16T10:00:45,234 INFO [Time-limited test {}] master.ServerManager(983): Cluster shutdown requested of master=037c58a19ddb,37877,1731751077131 2024-11-16T10:00:45,235 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:37877-0x101b083d7320000, quorum=127.0.0.1:49647, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running 2024-11-16T10:00:45,235 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:37267-0x101b083d7320001, quorum=127.0.0.1:49647, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running 2024-11-16T10:00:45,235 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:37877-0x101b083d7320000, quorum=127.0.0.1:49647, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-16T10:00:45,235 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:37267-0x101b083d7320001, quorum=127.0.0.1:49647, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-16T10:00:45,235 INFO [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-16T10:00:45,236 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-16T10:00:45,236 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.master.HMaster.lambda$shutdown$17(HMaster.java:3306) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.master.HMaster.shutdown(HMaster.java:3277) at org.apache.hadoop.hbase.util.JVMClusterUtil.shutdown(JVMClusterUtil.java:265) at org.apache.hadoop.hbase.LocalHBaseCluster.shutdown(LocalHBaseCluster.java:416) at org.apache.hadoop.hbase.SingleProcessHBaseCluster.shutdown(SingleProcessHBaseCluster.java:676) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniHBaseCluster(HBaseTestingUtil.java:1036) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1020) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.tearDownAfterClass(AcidGuaranteesTestBase.java:66) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-16T10:00:45,236 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T10:00:45,236 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): regionserver:37267-0x101b083d7320001, quorum=127.0.0.1:49647, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-16T10:00:45,236 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): master:37877-0x101b083d7320000, quorum=127.0.0.1:49647, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-16T10:00:45,236 INFO [Time-limited test {}] regionserver.HRegionServer(2196): ***** STOPPING region server '037c58a19ddb,37267,1731751077844' ***** 2024-11-16T10:00:45,237 INFO [Time-limited test {}] regionserver.HRegionServer(2210): STOPPED: Shutdown requested 2024-11-16T10:00:45,237 INFO [RS:0;037c58a19ddb:37267 {}] regionserver.HeapMemoryManager(220): Stopping 2024-11-16T10:00:45,237 INFO [MemStoreFlusher.0 {}] regionserver.MemStoreFlusher$FlushHandler(374): MemStoreFlusher.0 exiting 2024-11-16T10:00:45,237 INFO [RS:0;037c58a19ddb:37267 {}] flush.RegionServerFlushTableProcedureManager(119): Stopping region server flush procedure manager gracefully. 2024-11-16T10:00:45,238 INFO [RS:0;037c58a19ddb:37267 {}] snapshot.RegionServerSnapshotManager(137): Stopping RegionServerSnapshotManager gracefully. 2024-11-16T10:00:45,238 INFO [RS:0;037c58a19ddb:37267 {}] regionserver.HRegionServer(959): stopping server 037c58a19ddb,37267,1731751077844 2024-11-16T10:00:45,238 INFO [RS:0;037c58a19ddb:37267 {}] hbase.HBaseServerBase(455): Close async cluster connection 2024-11-16T10:00:45,238 INFO [RS:0;037c58a19ddb:37267 {}] client.AsyncConnectionImpl(233): Connection has been closed by RS:0;037c58a19ddb:37267. 2024-11-16T10:00:45,238 DEBUG [RS:0;037c58a19ddb:37267 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.HBaseServerBase.closeClusterConnection(HBaseServerBase.java:457) at org.apache.hadoop.hbase.regionserver.HRegionServer.run(HRegionServer.java:962) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.runRegionServer(SingleProcessHBaseCluster.java:171) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer$1.run(SingleProcessHBaseCluster.java:155) at java.base/java.security.AccessController.doPrivileged(AccessController.java:399) at java.base/javax.security.auth.Subject.doAs(Subject.java:376) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1930) at org.apache.hadoop.hbase.security.User$SecureHadoopUser.runAs(User.java:322) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.run(SingleProcessHBaseCluster.java:152) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-16T10:00:45,238 DEBUG [RS:0;037c58a19ddb:37267 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T10:00:45,238 INFO [RS:0;037c58a19ddb:37267 {}] regionserver.CompactSplit(469): Waiting for Split Thread to finish... 2024-11-16T10:00:45,238 INFO [RS:0;037c58a19ddb:37267 {}] regionserver.CompactSplit(469): Waiting for Large Compaction Thread to finish... 2024-11-16T10:00:45,238 INFO [RS:0;037c58a19ddb:37267 {}] regionserver.CompactSplit(469): Waiting for Small Compaction Thread to finish... 2024-11-16T10:00:45,238 INFO [RS:0;037c58a19ddb:37267 {}] regionserver.HRegionServer(3091): Received CLOSE for 1588230740 2024-11-16T10:00:45,239 INFO [RS:0;037c58a19ddb:37267 {}] regionserver.HRegionServer(1321): Waiting on 1 regions to close 2024-11-16T10:00:45,239 DEBUG [RS:0;037c58a19ddb:37267 {}] regionserver.HRegionServer(1325): Online Regions={1588230740=hbase:meta,,1.1588230740} 2024-11-16T10:00:45,239 DEBUG [RS_CLOSE_META-regionserver/037c58a19ddb:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1722): Closing 1588230740, disabling compactions & flushes 2024-11-16T10:00:45,239 INFO [RS_CLOSE_META-regionserver/037c58a19ddb:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1755): Closing region hbase:meta,,1.1588230740 2024-11-16T10:00:45,239 DEBUG [RS_CLOSE_META-regionserver/037c58a19ddb:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:meta,,1.1588230740 2024-11-16T10:00:45,239 DEBUG [RS_CLOSE_META-regionserver/037c58a19ddb:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1843): Acquired close lock on hbase:meta,,1.1588230740 after waiting 0 ms 2024-11-16T10:00:45,239 DEBUG [RS_CLOSE_META-regionserver/037c58a19ddb:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1853): Updates disabled for region hbase:meta,,1.1588230740 2024-11-16T10:00:45,239 DEBUG [RS:0;037c58a19ddb:37267 {}] regionserver.HRegionServer(1351): Waiting on 1588230740 2024-11-16T10:00:45,239 INFO [RS_CLOSE_META-regionserver/037c58a19ddb:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(2902): Flushing 1588230740 4/4 column families, dataSize=19.91 KB heapSize=35.20 KB 2024-11-16T10:00:45,260 DEBUG [RS_CLOSE_META-regionserver/037c58a19ddb:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/hbase/meta/1588230740/.tmp/info/aa7eb8c2576a4194bb78d9463dd73f4c is 95, key is TestAcidGuarantees,,1731751082172.d9e28756f78b0555a6733ffbf6ab98bd./info:/1731751108923/DeleteFamily/seqid=0 2024-11-16T10:00:45,263 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37591 is added to blk_1073742471_1647 (size=6514) 2024-11-16T10:00:45,336 INFO [regionserver/037c58a19ddb:0.leaseChecker {}] regionserver.LeaseManager(133): Closed leases 2024-11-16T10:00:45,440 DEBUG [RS:0;037c58a19ddb:37267 {}] regionserver.HRegionServer(1351): Waiting on 1588230740 2024-11-16T10:00:45,640 DEBUG [RS:0;037c58a19ddb:37267 {}] regionserver.HRegionServer(1351): Waiting on 1588230740 2024-11-16T10:00:45,655 INFO [regionserver/037c58a19ddb:0.Chore.1 {}] hbase.ScheduledChore(149): Chore: CompactionChecker was stopped 2024-11-16T10:00:45,655 INFO [regionserver/037c58a19ddb:0.Chore.1 {}] hbase.ScheduledChore(149): Chore: MemstoreFlusherChore was stopped 2024-11-16T10:00:45,664 INFO [RS_CLOSE_META-regionserver/037c58a19ddb:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=16.75 KB at sequenceid=90 (bloomFilter=true), to=hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/hbase/meta/1588230740/.tmp/info/aa7eb8c2576a4194bb78d9463dd73f4c 2024-11-16T10:00:45,682 DEBUG [RS_CLOSE_META-regionserver/037c58a19ddb:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/hbase/meta/1588230740/.tmp/ns/feece7f097604816ba27a206f5d5f54a is 93, key is TestAcidGuarantees,,1731751082172.d9e28756f78b0555a6733ffbf6ab98bd./ns:/1731751108883/DeleteFamily/seqid=0 2024-11-16T10:00:45,684 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37591 is added to blk_1073742472_1648 (size=5906) 2024-11-16T10:00:45,840 DEBUG [RS:0;037c58a19ddb:37267 {}] regionserver.HRegionServer(1351): Waiting on 1588230740 2024-11-16T10:00:46,040 DEBUG [RS:0;037c58a19ddb:37267 {}] regionserver.HRegionServer(1351): Waiting on 1588230740 2024-11-16T10:00:46,085 INFO [RS_CLOSE_META-regionserver/037c58a19ddb:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=608 B at sequenceid=90 (bloomFilter=true), to=hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/hbase/meta/1588230740/.tmp/ns/feece7f097604816ba27a206f5d5f54a 2024-11-16T10:00:46,102 DEBUG [RS_CLOSE_META-regionserver/037c58a19ddb:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/hbase/meta/1588230740/.tmp/rep_barrier/f3f6403483d34cbfa4062c2a9df22790 is 102, key is TestAcidGuarantees,,1731751082172.d9e28756f78b0555a6733ffbf6ab98bd./rep_barrier:/1731751108883/DeleteFamily/seqid=0 2024-11-16T10:00:46,105 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37591 is added to blk_1073742473_1649 (size=6025) 2024-11-16T10:00:46,240 DEBUG [RS:0;037c58a19ddb:37267 {}] regionserver.HRegionServer(1351): Waiting on 1588230740 2024-11-16T10:00:46,441 DEBUG [RS:0;037c58a19ddb:37267 {}] regionserver.HRegionServer(1351): Waiting on 1588230740 2024-11-16T10:00:46,505 INFO [RS_CLOSE_META-regionserver/037c58a19ddb:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=588 B at sequenceid=90 (bloomFilter=true), to=hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/hbase/meta/1588230740/.tmp/rep_barrier/f3f6403483d34cbfa4062c2a9df22790 2024-11-16T10:00:46,523 DEBUG [RS_CLOSE_META-regionserver/037c58a19ddb:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/hbase/meta/1588230740/.tmp/table/eda34f851ac14cdf939988134c401c0a is 96, key is TestAcidGuarantees,,1731751082172.d9e28756f78b0555a6733ffbf6ab98bd./table:/1731751108883/DeleteFamily/seqid=0 2024-11-16T10:00:46,526 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37591 is added to blk_1073742474_1650 (size=5936) 2024-11-16T10:00:46,641 DEBUG [RS:0;037c58a19ddb:37267 {}] regionserver.HRegionServer(1351): Waiting on 1588230740 2024-11-16T10:00:46,841 DEBUG [RS:0;037c58a19ddb:37267 {}] regionserver.HRegionServer(1351): Waiting on 1588230740 2024-11-16T10:00:46,926 INFO [RS_CLOSE_META-regionserver/037c58a19ddb:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=1.99 KB at sequenceid=90 (bloomFilter=true), to=hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/hbase/meta/1588230740/.tmp/table/eda34f851ac14cdf939988134c401c0a 2024-11-16T10:00:46,931 DEBUG [RS_CLOSE_META-regionserver/037c58a19ddb:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/hbase/meta/1588230740/.tmp/info/aa7eb8c2576a4194bb78d9463dd73f4c as hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/hbase/meta/1588230740/info/aa7eb8c2576a4194bb78d9463dd73f4c 2024-11-16T10:00:46,934 INFO [RS_CLOSE_META-regionserver/037c58a19ddb:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/hbase/meta/1588230740/info/aa7eb8c2576a4194bb78d9463dd73f4c, entries=12, sequenceid=90, filesize=6.4 K 2024-11-16T10:00:46,935 DEBUG [RS_CLOSE_META-regionserver/037c58a19ddb:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/hbase/meta/1588230740/.tmp/ns/feece7f097604816ba27a206f5d5f54a as hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/hbase/meta/1588230740/ns/feece7f097604816ba27a206f5d5f54a 2024-11-16T10:00:46,939 INFO [RS_CLOSE_META-regionserver/037c58a19ddb:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/hbase/meta/1588230740/ns/feece7f097604816ba27a206f5d5f54a, entries=8, sequenceid=90, filesize=5.8 K 2024-11-16T10:00:46,939 DEBUG [RS_CLOSE_META-regionserver/037c58a19ddb:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/hbase/meta/1588230740/.tmp/rep_barrier/f3f6403483d34cbfa4062c2a9df22790 as hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/hbase/meta/1588230740/rep_barrier/f3f6403483d34cbfa4062c2a9df22790 2024-11-16T10:00:46,942 INFO [RS_CLOSE_META-regionserver/037c58a19ddb:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/hbase/meta/1588230740/rep_barrier/f3f6403483d34cbfa4062c2a9df22790, entries=6, sequenceid=90, filesize=5.9 K 2024-11-16T10:00:46,943 DEBUG [RS_CLOSE_META-regionserver/037c58a19ddb:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/hbase/meta/1588230740/.tmp/table/eda34f851ac14cdf939988134c401c0a as hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/hbase/meta/1588230740/table/eda34f851ac14cdf939988134c401c0a 2024-11-16T10:00:46,946 INFO [RS_CLOSE_META-regionserver/037c58a19ddb:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/hbase/meta/1588230740/table/eda34f851ac14cdf939988134c401c0a, entries=7, sequenceid=90, filesize=5.8 K 2024-11-16T10:00:46,947 INFO [RS_CLOSE_META-regionserver/037c58a19ddb:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(3140): Finished flush of dataSize ~19.91 KB/20384, heapSize ~35.14 KB/35984, currentSize=0 B/0 for 1588230740 in 1707ms, sequenceid=90, compaction requested=false 2024-11-16T10:00:46,954 DEBUG [RS_CLOSE_META-regionserver/037c58a19ddb:0-0 {event_type=M_RS_CLOSE_META}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/data/hbase/meta/1588230740/recovered.edits/93.seqid, newMaxSeqId=93, maxSeqId=1 2024-11-16T10:00:46,955 DEBUG [RS_CLOSE_META-regionserver/037c58a19ddb:0-0 {event_type=M_RS_CLOSE_META}] coprocessor.CoprocessorHost(310): Stop coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint 2024-11-16T10:00:46,955 INFO [RS_CLOSE_META-regionserver/037c58a19ddb:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1973): Closed hbase:meta,,1.1588230740 2024-11-16T10:00:46,955 DEBUG [RS_CLOSE_META-regionserver/037c58a19ddb:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1676): Region close journal for 1588230740: Waiting for close lock at 1731751245239Running coprocessor pre-close hooks at 1731751245239Disabling compacts and flushes for region at 1731751245239Disabling writes for close at 1731751245239Obtaining lock to block concurrent updates at 1731751245239Preparing flush snapshotting stores in 1588230740 at 1731751245239Finished memstore snapshotting hbase:meta,,1.1588230740, syncing WAL and waiting on mvcc, flushsize=dataSize=20384, getHeapSize=35984, getOffHeapSize=0, getCellsCount=191 at 1731751245240 (+1 ms)Flushing stores of hbase:meta,,1.1588230740 at 1731751245240Flushing 1588230740/info: creating writer at 1731751245240Flushing 1588230740/info: appending metadata at 1731751245259 (+19 ms)Flushing 1588230740/info: closing flushed file at 1731751245260 (+1 ms)Flushing 1588230740/ns: creating writer at 1731751245667 (+407 ms)Flushing 1588230740/ns: appending metadata at 1731751245681 (+14 ms)Flushing 1588230740/ns: closing flushed file at 1731751245682 (+1 ms)Flushing 1588230740/rep_barrier: creating writer at 1731751246088 (+406 ms)Flushing 1588230740/rep_barrier: appending metadata at 1731751246102 (+14 ms)Flushing 1588230740/rep_barrier: closing flushed file at 1731751246102Flushing 1588230740/table: creating writer at 1731751246508 (+406 ms)Flushing 1588230740/table: appending metadata at 1731751246523 (+15 ms)Flushing 1588230740/table: closing flushed file at 1731751246523Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@309e2d35: reopening flushed file at 1731751246930 (+407 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@3d5a8337: reopening flushed file at 1731751246934 (+4 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@786cdf4e: reopening flushed file at 1731751246939 (+5 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@2e3ad0fc: reopening flushed file at 1731751246942 (+3 ms)Finished flush of dataSize ~19.91 KB/20384, heapSize ~35.14 KB/35984, currentSize=0 B/0 for 1588230740 in 1707ms, sequenceid=90, compaction requested=false at 1731751246947 (+5 ms)Writing region close event to WAL at 1731751246952 (+5 ms)Running coprocessor post-close hooks at 1731751246955 (+3 ms)Closed at 1731751246955 2024-11-16T10:00:46,955 DEBUG [RS_CLOSE_META-regionserver/037c58a19ddb:0-0 {event_type=M_RS_CLOSE_META}] handler.CloseRegionHandler(117): Closed hbase:meta,,1.1588230740 2024-11-16T10:00:47,041 INFO [RS:0;037c58a19ddb:37267 {}] regionserver.HRegionServer(976): stopping server 037c58a19ddb,37267,1731751077844; all regions closed. 2024-11-16T10:00:47,047 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37591 is added to blk_1073741834_1010 (size=25344) 2024-11-16T10:00:47,050 DEBUG [RS:0;037c58a19ddb:37267 {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/oldWALs 2024-11-16T10:00:47,050 INFO [RS:0;037c58a19ddb:37267 {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL 037c58a19ddb%2C37267%2C1731751077844.meta:.meta(num 1731751081722) 2024-11-16T10:00:47,051 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37591 is added to blk_1073741832_1008 (size=14868637) 2024-11-16T10:00:47,053 DEBUG [RS:0;037c58a19ddb:37267 {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/oldWALs 2024-11-16T10:00:47,053 INFO [RS:0;037c58a19ddb:37267 {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL 037c58a19ddb%2C37267%2C1731751077844:(num 1731751080812) 2024-11-16T10:00:47,053 DEBUG [RS:0;037c58a19ddb:37267 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T10:00:47,053 INFO [RS:0;037c58a19ddb:37267 {}] regionserver.LeaseManager(133): Closed leases 2024-11-16T10:00:47,053 INFO [RS:0;037c58a19ddb:37267 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service 2024-11-16T10:00:47,053 INFO [RS:0;037c58a19ddb:37267 {}] hbase.ChoreService(370): Chore service for: regionserver/037c58a19ddb:0 had [ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS, ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS, ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS] on shutdown 2024-11-16T10:00:47,053 INFO [RS:0;037c58a19ddb:37267 {}] hbase.HBaseServerBase(448): Shutdown executor service 2024-11-16T10:00:47,053 INFO [regionserver/037c58a19ddb:0.logRoller {}] wal.AbstractWALRoller(249): LogRoller exiting. 2024-11-16T10:00:47,054 INFO [RS:0;037c58a19ddb:37267 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.2:37267 2024-11-16T10:00:47,058 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:37267-0x101b083d7320001, quorum=127.0.0.1:49647, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/rs/037c58a19ddb,37267,1731751077844 2024-11-16T10:00:47,058 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:37877-0x101b083d7320000, quorum=127.0.0.1:49647, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-11-16T10:00:47,058 INFO [RS:0;037c58a19ddb:37267 {}] hbase.HBaseServerBase(479): Close zookeeper 2024-11-16T10:00:47,060 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(171): RegionServer ephemeral node deleted, processing expiration [037c58a19ddb,37267,1731751077844] 2024-11-16T10:00:47,062 DEBUG [RegionServerTracker-0 {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/draining/037c58a19ddb,37267,1731751077844 already deleted, retry=false 2024-11-16T10:00:47,062 INFO [RegionServerTracker-0 {}] master.ServerManager(688): Cluster shutdown set; 037c58a19ddb,37267,1731751077844 expired; onlineServers=0 2024-11-16T10:00:47,062 INFO [RegionServerTracker-0 {}] master.HMaster(3321): ***** STOPPING master '037c58a19ddb,37877,1731751077131' ***** 2024-11-16T10:00:47,062 INFO [RegionServerTracker-0 {}] master.HMaster(3323): STOPPED: Cluster shutdown set; onlineServer=0 2024-11-16T10:00:47,062 INFO [M:0;037c58a19ddb:37877 {}] hbase.HBaseServerBase(455): Close async cluster connection 2024-11-16T10:00:47,062 INFO [M:0;037c58a19ddb:37877 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service 2024-11-16T10:00:47,062 DEBUG [M:0;037c58a19ddb:37877 {}] cleaner.LogCleaner(198): Cancelling LogCleaner 2024-11-16T10:00:47,062 DEBUG [M:0;037c58a19ddb:37877 {}] cleaner.HFileCleaner(335): Stopping file delete threads 2024-11-16T10:00:47,062 WARN [OldWALsCleaner-0 {}] cleaner.LogCleaner(186): Interrupted while cleaning old WALs, will try to clean it next round. Exiting. 2024-11-16T10:00:47,062 DEBUG [master/037c58a19ddb:0:becomeActiveMaster-HFileCleaner.large.0-1731751080462 {}] cleaner.HFileCleaner(306): Exit Thread[master/037c58a19ddb:0:becomeActiveMaster-HFileCleaner.large.0-1731751080462,5,FailOnTimeoutGroup] 2024-11-16T10:00:47,062 DEBUG [master/037c58a19ddb:0:becomeActiveMaster-HFileCleaner.small.0-1731751080467 {}] cleaner.HFileCleaner(306): Exit Thread[master/037c58a19ddb:0:becomeActiveMaster-HFileCleaner.small.0-1731751080467,5,FailOnTimeoutGroup] 2024-11-16T10:00:47,063 INFO [M:0;037c58a19ddb:37877 {}] hbase.ChoreService(370): Chore service for: master/037c58a19ddb:0 had [ScheduledChore name=FlushedSequenceIdFlusher, period=10800000, unit=MILLISECONDS] on shutdown 2024-11-16T10:00:47,063 INFO [M:0;037c58a19ddb:37877 {}] hbase.HBaseServerBase(448): Shutdown executor service 2024-11-16T10:00:47,063 DEBUG [M:0;037c58a19ddb:37877 {}] master.HMaster(1795): Stopping service threads 2024-11-16T10:00:47,063 INFO [M:0;037c58a19ddb:37877 {}] procedure2.RemoteProcedureDispatcher(119): Stopping procedure remote dispatcher 2024-11-16T10:00:47,063 INFO [M:0;037c58a19ddb:37877 {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-16T10:00:47,063 ERROR [M:0;037c58a19ddb:37877 {}] procedure2.ProcedureExecutor(763): There are still active thread in group java.lang.ThreadGroup[name=PEWorkerGroup,maxpri=10], see STDOUT java.lang.ThreadGroup[name=PEWorkerGroup,maxpri=10] Thread[HFileArchiver-5,5,PEWorkerGroup] Thread[IPC Client (1102635514) connection to localhost/127.0.0.1:44755 from jenkins,5,PEWorkerGroup] Thread[IPC Parameter Sending Thread for localhost/127.0.0.1:44755,5,PEWorkerGroup] Thread[HFileArchiver-6,5,PEWorkerGroup] 2024-11-16T10:00:47,064 INFO [M:0;037c58a19ddb:37877 {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false 2024-11-16T10:00:47,064 DEBUG [normalizer-worker-0 {}] normalizer.RegionNormalizerWorker(193): interrupt detected. terminating. 2024-11-16T10:00:47,064 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:37877-0x101b083d7320000, quorum=127.0.0.1:49647, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/master 2024-11-16T10:00:47,064 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:37877-0x101b083d7320000, quorum=127.0.0.1:49647, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-16T10:00:47,065 DEBUG [M:0;037c58a19ddb:37877 {}] zookeeper.ZKUtil(347): master:37877-0x101b083d7320000, quorum=127.0.0.1:49647, baseZNode=/hbase Unable to get data of znode /hbase/master because node does not exist (not an error) 2024-11-16T10:00:47,065 WARN [M:0;037c58a19ddb:37877 {}] master.ActiveMasterManager(344): Failed get of master address: java.io.IOException: Can't get master address from ZooKeeper; znode data == null 2024-11-16T10:00:47,065 INFO [M:0;037c58a19ddb:37877 {}] master.ServerManager(1139): Writing .lastflushedseqids file at: hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/.lastflushedseqids 2024-11-16T10:00:47,076 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37591 is added to blk_1073742475_1651 (size=72) 2024-11-16T10:00:47,160 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:37267-0x101b083d7320001, quorum=127.0.0.1:49647, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-16T10:00:47,160 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:37267-0x101b083d7320001, quorum=127.0.0.1:49647, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-16T10:00:47,161 INFO [RS:0;037c58a19ddb:37267 {}] hbase.HBaseServerBase(486): Close table descriptors 2024-11-16T10:00:47,161 INFO [RS:0;037c58a19ddb:37267 {}] regionserver.HRegionServer(1031): Exiting; stopping=037c58a19ddb,37267,1731751077844; zookeeper connection closed. 2024-11-16T10:00:47,161 INFO [Shutdown of org.apache.hadoop.hbase.fs.HFileSystem@624ee9a6 {}] hbase.SingleProcessHBaseCluster$SingleFileSystemShutdownThread(211): Hook closing fs=org.apache.hadoop.hbase.fs.HFileSystem@624ee9a6 2024-11-16T10:00:47,162 INFO [Time-limited test {}] util.JVMClusterUtil(335): Shutdown of 1 master(s) and 1 regionserver(s) complete 2024-11-16T10:00:47,477 INFO [M:0;037c58a19ddb:37877 {}] assignment.AssignmentManager(395): Stopping assignment manager 2024-11-16T10:00:47,477 INFO [M:0;037c58a19ddb:37877 {}] region.MasterRegion(195): Closing local region {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}, isAbort=false 2024-11-16T10:00:47,477 DEBUG [M:0;037c58a19ddb:37877 {}] regionserver.HRegion(1722): Closing 1595e783b53d99cd5eef43b6debb2682, disabling compactions & flushes 2024-11-16T10:00:47,477 INFO [M:0;037c58a19ddb:37877 {}] regionserver.HRegion(1755): Closing region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-16T10:00:47,477 DEBUG [M:0;037c58a19ddb:37877 {}] regionserver.HRegion(1776): Time limited wait for close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-16T10:00:47,477 DEBUG [M:0;037c58a19ddb:37877 {}] regionserver.HRegion(1843): Acquired close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. after waiting 0 ms 2024-11-16T10:00:47,477 DEBUG [M:0;037c58a19ddb:37877 {}] regionserver.HRegion(1853): Updates disabled for region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-16T10:00:47,477 INFO [M:0;037c58a19ddb:37877 {}] regionserver.HRegion(2902): Flushing 1595e783b53d99cd5eef43b6debb2682 4/4 column families, dataSize=761.58 KB heapSize=936.50 KB 2024-11-16T10:00:47,492 DEBUG [M:0;037c58a19ddb:37877 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/3aa2ac05f6544aa9a10ea0f4dd813151 is 82, key is hbase:meta,,1/info:regioninfo/1731751081806/Put/seqid=0 2024-11-16T10:00:47,495 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37591 is added to blk_1073742476_1652 (size=5672) 2024-11-16T10:00:47,539 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(153): Removing adapter for the MetricRegistry: RegionServer,sub=Coprocessor.Region.CP_org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint 2024-11-16T10:00:47,539 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(153): Removing adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_hbase_table_meta 2024-11-16T10:00:47,539 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(153): Removing adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_TestAcidGuarantees 2024-11-16T10:00:47,895 INFO [M:0;037c58a19ddb:37877 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=504 B at sequenceid=2162 (bloomFilter=true), to=hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/3aa2ac05f6544aa9a10ea0f4dd813151 2024-11-16T10:00:47,916 DEBUG [M:0;037c58a19ddb:37877 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/e79eb49951074c4aade2c8ec25f798f2 is 2277, key is \x00\x00\x00\x00\x00\x00\x00\x91/proc:d/1731751220388/Put/seqid=0 2024-11-16T10:00:47,920 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37591 is added to blk_1073742477_1653 (size=43856) 2024-11-16T10:00:48,320 INFO [M:0;037c58a19ddb:37877 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=761.03 KB at sequenceid=2162 (bloomFilter=true), to=hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/e79eb49951074c4aade2c8ec25f798f2 2024-11-16T10:00:48,324 INFO [M:0;037c58a19ddb:37877 {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for e79eb49951074c4aade2c8ec25f798f2 2024-11-16T10:00:48,338 DEBUG [M:0;037c58a19ddb:37877 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/033d098b4785497fb1ec347c90712b72 is 69, key is 037c58a19ddb,37267,1731751077844/rs:state/1731751080587/Put/seqid=0 2024-11-16T10:00:48,341 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37591 is added to blk_1073742478_1654 (size=5156) 2024-11-16T10:00:48,742 INFO [M:0;037c58a19ddb:37877 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=65 B at sequenceid=2162 (bloomFilter=true), to=hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/033d098b4785497fb1ec347c90712b72 2024-11-16T10:00:48,746 DEBUG [M:0;037c58a19ddb:37877 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/3aa2ac05f6544aa9a10ea0f4dd813151 as hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/info/3aa2ac05f6544aa9a10ea0f4dd813151 2024-11-16T10:00:48,749 INFO [M:0;037c58a19ddb:37877 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/info/3aa2ac05f6544aa9a10ea0f4dd813151, entries=8, sequenceid=2162, filesize=5.5 K 2024-11-16T10:00:48,749 DEBUG [M:0;037c58a19ddb:37877 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/e79eb49951074c4aade2c8ec25f798f2 as hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/proc/e79eb49951074c4aade2c8ec25f798f2 2024-11-16T10:00:48,752 INFO [M:0;037c58a19ddb:37877 {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for e79eb49951074c4aade2c8ec25f798f2 2024-11-16T10:00:48,752 INFO [M:0;037c58a19ddb:37877 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/proc/e79eb49951074c4aade2c8ec25f798f2, entries=168, sequenceid=2162, filesize=42.8 K 2024-11-16T10:00:48,752 DEBUG [M:0;037c58a19ddb:37877 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/033d098b4785497fb1ec347c90712b72 as hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/rs/033d098b4785497fb1ec347c90712b72 2024-11-16T10:00:48,755 INFO [M:0;037c58a19ddb:37877 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44755/user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/rs/033d098b4785497fb1ec347c90712b72, entries=1, sequenceid=2162, filesize=5.0 K 2024-11-16T10:00:48,756 INFO [M:0;037c58a19ddb:37877 {}] regionserver.HRegion(3140): Finished flush of dataSize ~761.58 KB/779862, heapSize ~936.20 KB/958672, currentSize=0 B/0 for 1595e783b53d99cd5eef43b6debb2682 in 1279ms, sequenceid=2162, compaction requested=false 2024-11-16T10:00:48,757 INFO [M:0;037c58a19ddb:37877 {}] regionserver.HRegion(1973): Closed master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-16T10:00:48,757 DEBUG [M:0;037c58a19ddb:37877 {}] regionserver.HRegion(1676): Region close journal for 1595e783b53d99cd5eef43b6debb2682: Waiting for close lock at 1731751247477Disabling compacts and flushes for region at 1731751247477Disabling writes for close at 1731751247477Obtaining lock to block concurrent updates at 1731751247477Preparing flush snapshotting stores in 1595e783b53d99cd5eef43b6debb2682 at 1731751247477Finished memstore snapshotting master:store,,1.1595e783b53d99cd5eef43b6debb2682., syncing WAL and waiting on mvcc, flushsize=dataSize=779862, getHeapSize=958912, getOffHeapSize=0, getCellsCount=2355 at 1731751247477Flushing stores of master:store,,1.1595e783b53d99cd5eef43b6debb2682. at 1731751247478 (+1 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/info: creating writer at 1731751247478Flushing 1595e783b53d99cd5eef43b6debb2682/info: appending metadata at 1731751247492 (+14 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/info: closing flushed file at 1731751247492Flushing 1595e783b53d99cd5eef43b6debb2682/proc: creating writer at 1731751247899 (+407 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/proc: appending metadata at 1731751247916 (+17 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/proc: closing flushed file at 1731751247916Flushing 1595e783b53d99cd5eef43b6debb2682/rs: creating writer at 1731751248324 (+408 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/rs: appending metadata at 1731751248338 (+14 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/rs: closing flushed file at 1731751248338Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@27eaaf1e: reopening flushed file at 1731751248745 (+407 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@56e1132c: reopening flushed file at 1731751248749 (+4 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@16d10000: reopening flushed file at 1731751248752 (+3 ms)Finished flush of dataSize ~761.58 KB/779862, heapSize ~936.20 KB/958672, currentSize=0 B/0 for 1595e783b53d99cd5eef43b6debb2682 in 1279ms, sequenceid=2162, compaction requested=false at 1731751248756 (+4 ms)Writing region close event to WAL at 1731751248757 (+1 ms)Closed at 1731751248757 2024-11-16T10:00:48,759 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37591 is added to blk_1073741830_1006 (size=921583) 2024-11-16T10:00:48,760 WARN [Close-WAL-Writer-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(650): complete file /user/jenkins/test-data/7331a424-85b6-d3af-e471-9aedaf7d5592/MasterData/WALs/037c58a19ddb,37877,1731751077131/037c58a19ddb%2C37877%2C1731751077131.1731751079922 not finished, retry = 0 2024-11-16T10:00:48,861 INFO [master:store-WAL-Roller {}] wal.AbstractWALRoller(249): LogRoller exiting. 2024-11-16T10:00:48,861 INFO [M:0;037c58a19ddb:37877 {}] flush.MasterFlushTableProcedureManager(90): stop: server shutting down. 2024-11-16T10:00:48,861 INFO [M:0;037c58a19ddb:37877 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.2:37877 2024-11-16T10:00:48,861 INFO [M:0;037c58a19ddb:37877 {}] hbase.HBaseServerBase(479): Close zookeeper 2024-11-16T10:00:48,964 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:37877-0x101b083d7320000, quorum=127.0.0.1:49647, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-16T10:00:48,964 INFO [M:0;037c58a19ddb:37877 {}] hbase.HBaseServerBase(486): Close table descriptors 2024-11-16T10:00:48,964 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:37877-0x101b083d7320000, quorum=127.0.0.1:49647, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-16T10:00:48,969 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@4269bd31{datanode,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode} 2024-11-16T10:00:48,971 INFO [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@353d64f2{HTTP/1.1, (http/1.1)}{localhost:0} 2024-11-16T10:00:48,971 INFO [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging 2024-11-16T10:00:48,971 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@435f9f96{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED} 2024-11-16T10:00:48,972 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@4fd7fa7c{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/62476c2b-1e2d-7029-5a2f-94eb6a0fedba/hadoop.log.dir/,STOPPED} 2024-11-16T10:00:48,974 ERROR [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1416): Command processor encountered interrupt and exit. 2024-11-16T10:00:48,974 WARN [BP-1247346184-172.17.0.2-1731751074355 heartbeating to localhost/127.0.0.1:44755 {}] datanode.IncrementalBlockReportManager(160): IncrementalBlockReportManager interrupted 2024-11-16T10:00:48,975 WARN [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1400): Ending command processor service for: Thread[Command processor,5,FailOnTimeoutGroup] 2024-11-16T10:00:48,975 WARN [BP-1247346184-172.17.0.2-1731751074355 heartbeating to localhost/127.0.0.1:44755 {}] datanode.BPServiceActor(925): Ending block pool service for: Block pool BP-1247346184-172.17.0.2-1731751074355 (Datanode Uuid 347a0a29-1ecd-4397-8217-aa195b8a4ba6) service to localhost/127.0.0.1:44755 2024-11-16T10:00:48,977 WARN [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/62476c2b-1e2d-7029-5a2f-94eb6a0fedba/cluster_7e615273-6ed6-1343-5769-ac1955575909/data/data1/current/BP-1247346184-172.17.0.2-1731751074355 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted 2024-11-16T10:00:48,977 WARN [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/62476c2b-1e2d-7029-5a2f-94eb6a0fedba/cluster_7e615273-6ed6-1343-5769-ac1955575909/data/data2/current/BP-1247346184-172.17.0.2-1731751074355 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted 2024-11-16T10:00:48,977 WARN [Time-limited test {}] datanode.DataSetLockManager(261): not open lock leak check func 2024-11-16T10:00:48,984 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@4ded99d3{hdfs,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/hdfs} 2024-11-16T10:00:48,985 INFO [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@557285fa{HTTP/1.1, (http/1.1)}{localhost:0} 2024-11-16T10:00:48,985 INFO [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging 2024-11-16T10:00:48,985 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@93c62df{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED} 2024-11-16T10:00:48,985 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@6b23711f{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/62476c2b-1e2d-7029-5a2f-94eb6a0fedba/hadoop.log.dir/,STOPPED} 2024-11-16T10:00:49,000 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(347): Shutdown MiniZK cluster with all ZK servers 2024-11-16T10:00:49,117 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1026): Minicluster is down